summaryrefslogtreecommitdiff
path: root/rhimport/fetcher.go
diff options
context:
space:
mode:
Diffstat (limited to 'rhimport/fetcher.go')
-rw-r--r--rhimport/fetcher.go234
1 files changed, 132 insertions, 102 deletions
diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go
index ea12806..1a40e71 100644
--- a/rhimport/fetcher.go
+++ b/rhimport/fetcher.go
@@ -44,6 +44,9 @@ import (
"github.com/andelf/go-curl"
)
+//****************************************************************
+//** cURL based importer
+
type fetcherCurlCBData struct {
ctx *Context
res *Result
@@ -129,6 +132,12 @@ func curlProgressCallback(dltotal, dlnow, ultotal, ulnow float64, userdata inter
return true
}
+//***********************
+//** http://
+//** https://
+//** ftp://
+//** ftps://
+
func checkYoutubeDL(ctx *Context, res *Result, uri *url.URL) *youtubeDLInfo {
cmd := exec.Command("youtube-dl", "--no-playlist", "-f", "bestaudio/best", "--prefer-free-formats", "-J", ctx.SourceUri)
var stderr, stdout bytes.Buffer
@@ -273,6 +282,9 @@ func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) {
return
}
+//***********************
+//** archiv://
+
var months = [...]string{
"Jänner",
"Februar",
@@ -385,34 +397,105 @@ func fetchFileArchiv(ctx *Context, res *Result, uri *url.URL) (err error) {
return
}
-func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) {
- ctx.stdlog.Printf("tmp fetcher called for '%s'", ctx.SourceUri)
+//****************************************************************
+//** attachment://
- ctx.SourceFile = filepath.Join(ctx.conf.TempDir, path.Clean("/"+uri.Path))
+func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv fetchConverter) error {
+ written := uint64(0)
+ for {
+ select {
+ case <-ctx.Cancel:
+ ctx.stdlog.Printf("receiving attachment '%s' got canceled", ctx.SourceFile)
+ res.ResponseCode = http.StatusNoContent
+ res.ErrorString = "canceled"
+ return nil
+ case chunk, ok := <-ctx.AttachmentChan:
+ if !ok {
+ ctx.stdlog.Printf("receiving attachment '%s' failed: channel has been closed prematurely, after %d Bytes", ctx.SourceFile, written)
+ res.ResponseCode = http.StatusBadRequest
+ res.ErrorString = fmt.Sprintf("file upload stopped prematurely (after %d Bytes)", written)
+ return nil
+ }
+ if chunk.Error != nil {
+ ctx.stdlog.Printf("receiving attachment '%s' failed: %v", ctx.SourceFile, chunk.Error)
+ res.ResponseCode = http.StatusInternalServerError
+ res.ErrorString = chunk.Error.Error()
+ return nil
+ }
- size := int64(0)
- if info, err := os.Stat(ctx.SourceFile); err != nil {
+ left := sizeTotal - written
+ if int(left) < len(chunk.Data) {
+ ctx.stdlog.Printf("attachment fetcher: truncating %d byes of extra data", len(chunk.Data)-int(left))
+ chunk.Data = chunk.Data[0:left]
+ }
+
+ w, err := conv.Write(chunk.Data)
+ if err != nil {
+ ctx.stdlog.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err)
+ return err
+ }
+ written += uint64(w)
+
+ ctx.reportProgress(1, "receiving", float64(written), float64(sizeTotal))
+ if uint64(written) >= sizeTotal {
+ return nil
+ }
+ }
+ }
+}
+
+func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error {
+ ctx.dbglog.Printf("Attachment fetcher for '%s'", uri.String())
+
+ sizeTotal, err := strconv.ParseUint(uri.Host, 10, 64)
+ if err != nil {
res.ResponseCode = http.StatusBadRequest
- res.ErrorString = fmt.Sprintf("local-file stat(): %s", err)
+ res.ErrorString = "invalid attachment size (must be a positive integer)"
return nil
- } else {
- size = info.Size()
- if info.IsDir() {
- res.ResponseCode = http.StatusBadRequest
- res.ErrorString = fmt.Sprintf("'%s' is a directory", ctx.SourceFile)
+ }
+ if sizeTotal > FILESIZE_MAX {
+ res.ResponseCode = http.StatusRequestEntityTooLarge
+ res.ErrorString = "file exceeds maximum file size"
+ return nil
+ }
+
+ ctx.SourceFile = filepath.Join(ctx.WorkDir, path.Clean("/"+uri.Path))
+
+ var conv fetchConverter
+ ctx.OrigFilename = ctx.SourceFile
+ if conv, ctx.SourceFile, err = newFetchConverter(ctx, ctx.SourceFile); err != nil {
+ ctx.stdlog.Printf("Unable to create converter for file %s: %s", ctx.OrigFilename, err)
+ return err
+ }
+
+ ctx.reportProgress(1, "receiving", 0.0, float64(sizeTotal))
+ err = writeAttachmentFile(ctx, res, sizeTotal, conv)
+ conv.Close()
+ if res.ResponseCode == http.StatusNoContent {
+ ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri)
+ return nil
+ }
+
+ ctx.dbglog.Printf("waiting for converter to finish...")
+ convOut, convErr := conv.GetResult(ctx, res)
+ if err != nil {
+ return err
+ }
+ if convErr != nil {
+ if res.ResponseCode == http.StatusNoContent {
+ ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri)
return nil
}
+ ctx.stdlog.Printf("converter error: %v; converter output: %s", convErr, convOut)
+ return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut)
}
- ctx.dbglog.Printf("1: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile)
- ctx.reportProgress(1, "fetching", 0.0, float64(size))
- oldWorkDir := filepath.Dir(ctx.SourceFile)
- ctx.dbglog.Printf("switching over to old workdir: %s", oldWorkDir)
- ctx.SwitchTempWorkDir(oldWorkDir)
- ctx.reportProgress(1, "fetching", float64(size), float64(size))
- ctx.dbglog.Printf("2: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile)
- return
+ ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr)
+ return nil
}
+//****************************************************************
+//** io.Reader based importer
+
func fetchFileConvert(ctx *Context, res *Result, origSrc io.Reader, sizeTotal int64) (err error) {
origDir, origFile := path.Split(ctx.SourceFile)
@@ -478,6 +561,9 @@ func fetchFileConvert(ctx *Context, res *Result, origSrc io.Reader, sizeTotal in
return
}
+//***********************
+//** local://
+
func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) {
ctx.stdlog.Printf("local fetcher called for '%s'", ctx.SourceUri)
@@ -513,6 +599,9 @@ func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) {
return fetchFileConvert(ctx, res, src, size)
}
+//***********************
+//** silence://
+
func fetchFileSilence(ctx *Context, res *Result, uri *url.URL) error {
ctx.dbglog.Printf("Silence fetcher for '%s'", ctx.SourceUri)
@@ -538,99 +627,40 @@ func fetchFileSilence(ctx *Context, res *Result, uri *url.URL) error {
return fetchFileConvert(ctx, res, wav, int64(size))
}
-func writeAttachmentFile(ctx *Context, res *Result, sizeTotal uint64, conv fetchConverter) error {
- written := uint64(0)
- for {
- select {
- case <-ctx.Cancel:
- ctx.stdlog.Printf("receiving attachment '%s' got canceled", ctx.SourceFile)
- res.ResponseCode = http.StatusNoContent
- res.ErrorString = "canceled"
- return nil
- case chunk, ok := <-ctx.AttachmentChan:
- if !ok {
- ctx.stdlog.Printf("receiving attachment '%s' failed: channel has been closed prematurely, after %d Bytes", ctx.SourceFile, written)
- res.ResponseCode = http.StatusBadRequest
- res.ErrorString = fmt.Sprintf("file upload stopped prematurely (after %d Bytes)", written)
- return nil
- }
- if chunk.Error != nil {
- ctx.stdlog.Printf("receiving attachment '%s' failed: %v", ctx.SourceFile, chunk.Error)
- res.ResponseCode = http.StatusInternalServerError
- res.ErrorString = chunk.Error.Error()
- return nil
- }
+//****************************************************************
+//** tmp://
- left := sizeTotal - written
- if int(left) < len(chunk.Data) {
- ctx.stdlog.Printf("attachment fetcher: truncating %d byes of extra data", len(chunk.Data)-int(left))
- chunk.Data = chunk.Data[0:left]
- }
-
- w, err := conv.Write(chunk.Data)
- if err != nil {
- ctx.stdlog.Printf("Unable to write to converter(%s): %s", ctx.SourceFile, err)
- return err
- }
- written += uint64(w)
-
- ctx.reportProgress(1, "receiving", float64(written), float64(sizeTotal))
- if uint64(written) >= sizeTotal {
- return nil
- }
- }
- }
-}
+func fetchFileTmp(ctx *Context, res *Result, uri *url.URL) (err error) {
+ ctx.stdlog.Printf("tmp fetcher called for '%s'", ctx.SourceUri)
-func fetchFileAttachment(ctx *Context, res *Result, uri *url.URL) error {
- ctx.dbglog.Printf("Attachment fetcher for '%s'", uri.String())
+ ctx.SourceFile = filepath.Join(ctx.conf.TempDir, path.Clean("/"+uri.Path))
- sizeTotal, err := strconv.ParseUint(uri.Host, 10, 64)
- if err != nil {
+ size := int64(0)
+ if info, err := os.Stat(ctx.SourceFile); err != nil {
res.ResponseCode = http.StatusBadRequest
- res.ErrorString = "invalid attachment size (must be a positive integer)"
- return nil
- }
- if sizeTotal > FILESIZE_MAX {
- res.ResponseCode = http.StatusRequestEntityTooLarge
- res.ErrorString = "file exceeds maximum file size"
- return nil
- }
-
- ctx.SourceFile = filepath.Join(ctx.WorkDir, path.Clean("/"+uri.Path))
-
- var conv fetchConverter
- ctx.OrigFilename = ctx.SourceFile
- if conv, ctx.SourceFile, err = newFetchConverter(ctx, ctx.SourceFile); err != nil {
- ctx.stdlog.Printf("Unable to create converter for file %s: %s", ctx.OrigFilename, err)
- return err
- }
-
- ctx.reportProgress(1, "receiving", 0.0, float64(sizeTotal))
- err = writeAttachmentFile(ctx, res, sizeTotal, conv)
- conv.Close()
- if res.ResponseCode == http.StatusNoContent {
- ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri)
+ res.ErrorString = fmt.Sprintf("local-file stat(): %s", err)
return nil
- }
-
- ctx.dbglog.Printf("waiting for converter to finish...")
- convOut, convErr := conv.GetResult(ctx, res)
- if err != nil {
- return err
- }
- if convErr != nil {
- if res.ResponseCode == http.StatusNoContent {
- ctx.stdlog.Printf("download of '%s' got canceled", ctx.SourceUri)
+ } else {
+ size = info.Size()
+ if info.IsDir() {
+ res.ResponseCode = http.StatusBadRequest
+ res.ErrorString = fmt.Sprintf("'%s' is a directory", ctx.SourceFile)
return nil
}
- ctx.stdlog.Printf("converter error: %v; converter output: %s", convErr, convOut)
- return fmt.Errorf("converter error: %v; converter output: %s", convErr, convOut)
}
- ctx.dbglog.Printf("converter: loudness correction = %.2f dB", ctx.LoudnessCorr)
- return nil
+ ctx.dbglog.Printf("1: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile)
+ ctx.reportProgress(1, "fetching", 0.0, float64(size))
+ oldWorkDir := filepath.Dir(ctx.SourceFile)
+ ctx.dbglog.Printf("switching over to old workdir: %s", oldWorkDir)
+ ctx.SwitchTempWorkDir(oldWorkDir)
+ ctx.reportProgress(1, "fetching", float64(size), float64(size))
+ ctx.dbglog.Printf("2: Title = '%s', OrigName = '%s', SourceFile = '%s'", ctx.Title, ctx.OrigFilename, ctx.SourceFile)
+ return
}
+//****************************************************************
+//** global stuff
+
type fetchFunc func(*Context, *Result, *url.URL) (err error)
var (