From e22f278ed72b8dbbe4174264d25c7eac54b077d1 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 3 Dec 2015 16:36:25 +0100 Subject: moved some code to seperate package diff --git a/log.go b/log.go new file mode 100644 index 0000000..46b0e52 --- /dev/null +++ b/log.go @@ -0,0 +1,38 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "io/ioutil" + "log" + "os" +) + +var ( + rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) + // use ioutil.Discard to switch that thing off + // rhtl = log.New(os.Stderr, "[rhdbg]\t", log.LstdFlags) + rhtl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) +) diff --git a/srvWeb.go b/srvWeb.go new file mode 100644 index 0000000..f51c8e8 --- /dev/null +++ b/srvWeb.go @@ -0,0 +1,41 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "fmt" + "html" + "net/http" + _ "net/http/pprof" +) + +func ServeWeb(addr_s string) { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path)) + }) + + rhl.Println("listening on", addr_s) + http.ListenAndServe(addr_s, nil) +} -- cgit v0.10.2 From e8f972b29d3cb2854de91b46d04f2a3c1d51b413 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 3 Dec 2015 19:59:01 +0100 Subject: moved control interface back to main package added importer at rhimport package diff --git a/importer.go b/importer.go new file mode 100644 index 0000000..5da6075 --- /dev/null +++ b/importer.go @@ -0,0 +1,99 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + // "bytes" + "fmt" + // "io" + // "mime/multipart" + // "net/http" + // "os" +) + +type ImportContext struct { + username string + groupname string + // TODO: password channel + cart int + channels int + normalization_level int + autotrim_level int + use_meta_data bool + source_file string + delete_source_file bool +} + +// func import_audio(url, file string) (err error) { + +// var b bytes.Buffer +// w := multipart.NewWriter(&b) + +// if err = w.WriteField("COMMAND", "2"); err != nil { +// return +// } +// if err = w.WriteField("LOGIN_NAME", ""); err != nil { +// return +// } +// if err = w.WriteField("PASSWORD", ""); err != nil { +// return +// } + +// f, err := os.Open(file) +// if err != nil { +// return +// } +// fw, err := w.CreateFormFile("FILENAME", file) +// if err != nil { +// return +// } +// if _, err = io.Copy(fw, f); err != nil { +// return +// } +// f.Close() + +// w.Close() + +// req, err := http.NewRequest("POST", url, &b) +// if err != nil { +// return +// } +// req.Header.Set("Content-Type", w.FormDataContentType()) + +// client := &http.Client{} +// res, err := client.Do(req) +// if err != nil { +// return +// } +// if res.StatusCode != http.StatusOK { +// err = fmt.Errorf("bad status: %s", res.Status) +// } +// return +// } + +func ImportFile(ctx *ImportContext) (err error) { + err = fmt.Errorf("not yet implemented") + return +} diff --git a/srvWeb.go b/srvWeb.go deleted file mode 100644 index f51c8e8..0000000 --- a/srvWeb.go +++ /dev/null @@ -1,41 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "fmt" - "html" - "net/http" - _ "net/http/pprof" -) - -func ServeWeb(addr_s string) { - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path)) - }) - - rhl.Println("listening on", addr_s) - http.ListenAndServe(addr_s, nil) -} -- cgit v0.10.2 From e3300689d20c70fa35e337a165f0f7a7deaf43b0 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 4 Dec 2015 01:26:12 +0100 Subject: introduced conf diff --git a/conf.go b/conf.go new file mode 100644 index 0000000..bd350c3 --- /dev/null +++ b/conf.go @@ -0,0 +1,98 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import () + +type getPasswordResult struct { + password string + err error +} + +type getPasswordRequest struct { + username string + cached bool + response chan getPasswordResult +} + +type Config struct { + configfile string + RDXportEndpoint string + db_host string + db_user string + db_passwd string + db_db string + // TODO: reference to sql connection + password_cache map[string]string + getPasswordChan chan getPasswordRequest + quit chan bool + done chan bool +} + +func (self Config) getPassword(username string, cached bool) (pwd string, err error) { + //TODO: actually fetch password from cache or DB + pwd = "12345" + return +} + +func (self Config) dispatchRequests() { + defer func() { self.done <- true }() + for { + select { + case <-self.quit: + return + case req := <-self.getPasswordChan: + if req.cached { + rhdl.Println("Config: got getPassword request for", req.username, "(cached)") + } else { + rhdl.Println("Config: got getPassword request for", req.username, "(not cached)") + } + pwd, err := self.getPassword(req.username, req.cached) + req.response <- getPasswordResult{pwd, err} + } + } +} + +func (self Config) Cleanup() { + self.quit <- true + <-self.done + close(self.quit) + close(self.done) + close(self.getPasswordChan) + //TODO : close db connection +} + +func NewConfig(configfile, rdxport_endpoint *string) (conf *Config, err error) { + conf = new(Config) + conf.configfile = *configfile + conf.quit = make(chan bool) + conf.done = make(chan bool) + conf.RDXportEndpoint = *rdxport_endpoint + //TODO : init db connection + conf.password_cache = make(map[string]string) + conf.getPasswordChan = make(chan getPasswordRequest) + go conf.dispatchRequests() + return +} diff --git a/importer.go b/importer.go index 5da6075..8c7e66e 100644 --- a/importer.go +++ b/importer.go @@ -34,16 +34,49 @@ import ( ) type ImportContext struct { - username string - groupname string - // TODO: password channel - cart int - channels int - normalization_level int - autotrim_level int - use_meta_data bool - source_file string - delete_source_file bool + Conf *Config + UserName string + password string + GroupName string + Cart int + Channels int + NormalizationLevel int + AutotrimLevel int + UseMetaData bool + SourceFile string + DeleteSourceFile bool +} + +func NewImportContext(conf *Config, user string, group string, cart int) *ImportContext { + ctx := new(ImportContext) + ctx.Conf = conf + ctx.UserName = user + ctx.GroupName = group + ctx.Cart = cart + ctx.Channels = 2 + ctx.NormalizationLevel = 1200 + ctx.AutotrimLevel = 0 + ctx.UseMetaData = false + ctx.SourceFile = "" + ctx.DeleteSourceFile = false + + return ctx +} + +func (ctx *ImportContext) getPassword(cached bool) (err error) { + req := getPasswordRequest{} + req.username = ctx.UserName + req.cached = cached + req.response = make(chan getPasswordResult) + ctx.Conf.getPasswordChan <- req + + res := <-req.response + if res.err != nil { + err = res.err + return + } + ctx.password = res.password + return } // func import_audio(url, file string) (err error) { @@ -94,6 +127,14 @@ type ImportContext struct { // } func ImportFile(ctx *ImportContext) (err error) { - err = fmt.Errorf("not yet implemented") + rhl.Println("ImportFile called for", ctx.SourceFile) + + if err = ctx.getPassword(true); err != nil { + return + } + rhdl.Println("credentials:", ctx.UserName, "/", ctx.password) + + err = fmt.Errorf("%+v", ctx) + return } diff --git a/log.go b/log.go index 46b0e52..b192c30 100644 --- a/log.go +++ b/log.go @@ -25,14 +25,13 @@ package rhimport import ( - "io/ioutil" + // "io/ioutil" "log" "os" ) var ( - rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) - // use ioutil.Discard to switch that thing off - // rhtl = log.New(os.Stderr, "[rhdbg]\t", log.LstdFlags) - rhtl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) + rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) + rhdl = log.New(os.Stderr, "[rhimport-dbg]\t", log.LstdFlags) + //rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) ) -- cgit v0.10.2 From 91e974bee77344e3a5b1517843388994aae5c26d Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 4 Dec 2015 01:56:43 +0100 Subject: reading rd.conf file works now diff --git a/conf.go b/conf.go index bd350c3..d482588 100644 --- a/conf.go +++ b/conf.go @@ -24,7 +24,9 @@ package rhimport -import () +import ( + "github.com/vaughan0/go-ini" +) type getPasswordResult struct { password string @@ -51,13 +53,34 @@ type Config struct { done chan bool } -func (self Config) getPassword(username string, cached bool) (pwd string, err error) { +func get_ini_value(file ini.File, section string, key string, dflt string) string { + value, ok := file.Get(section, key) + if ok { + return value + } + return dflt +} + +func (self *Config) read_config_file() error { + file, err := ini.LoadFile(self.configfile) + if err != nil { + return err + } + + self.db_host = get_ini_value(file, "mySQL", "Hostname", "localhost") + self.db_user = get_ini_value(file, "mySQL", "Loginname", "rivendell") + self.db_passwd = get_ini_value(file, "mySQL", "Password", "letmein") + self.db_db = get_ini_value(file, "mySQL", "Database", "rivendell") + return nil +} + +func (self *Config) getPassword(username string, cached bool) (pwd string, err error) { //TODO: actually fetch password from cache or DB pwd = "12345" return } -func (self Config) dispatchRequests() { +func (self *Config) dispatchRequests() { defer func() { self.done <- true }() for { select { @@ -75,7 +98,7 @@ func (self Config) dispatchRequests() { } } -func (self Config) Cleanup() { +func (self *Config) Cleanup() { self.quit <- true <-self.done close(self.quit) @@ -87,12 +110,18 @@ func (self Config) Cleanup() { func NewConfig(configfile, rdxport_endpoint *string) (conf *Config, err error) { conf = new(Config) conf.configfile = *configfile + err = conf.read_config_file() + if err != nil { + return + } conf.quit = make(chan bool) conf.done = make(chan bool) conf.RDXportEndpoint = *rdxport_endpoint - //TODO : init db connection conf.password_cache = make(map[string]string) conf.getPasswordChan = make(chan getPasswordRequest) + + //TODO : init db connection + go conf.dispatchRequests() return } -- cgit v0.10.2 From 8d71a7cf19dc3e36c0d9ac5a789ae647896859af Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 4 Dec 2015 03:01:18 +0100 Subject: fetching of user passwords works now diff --git a/conf.go b/conf.go index d482588..6dcbe84 100644 --- a/conf.go +++ b/conf.go @@ -25,6 +25,9 @@ package rhimport import ( + "database/sql" + "fmt" + _ "github.com/go-sql-driver/mysql" "github.com/vaughan0/go-ini" ) @@ -40,17 +43,18 @@ type getPasswordRequest struct { } type Config struct { - configfile string - RDXportEndpoint string - db_host string - db_user string - db_passwd string - db_db string - // TODO: reference to sql connection - password_cache map[string]string - getPasswordChan chan getPasswordRequest - quit chan bool - done chan bool + configfile string + RDXportEndpoint string + db_host string + db_user string + db_passwd string + db_db string + dbh *sql.DB + password_cache map[string]string + getPasswordChan chan getPasswordRequest + dbGetPasswordStmt *sql.Stmt + quit chan bool + done chan bool } func get_ini_value(file ini.File, section string, key string, dflt string) string { @@ -74,9 +78,35 @@ func (self *Config) read_config_file() error { return nil } +func (self *Config) init_database() (err error) { + dsn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", self.db_user, self.db_passwd, self.db_host, self.db_db) + if self.dbh, err = sql.Open("mysql", dsn); err != nil { + return + } + if self.dbGetPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?"); err != nil { + return + } + + return +} + func (self *Config) getPassword(username string, cached bool) (pwd string, err error) { - //TODO: actually fetch password from cache or DB - pwd = "12345" + + if cached { + pwd = self.password_cache[username] + } + + if pwd == "" { + err = self.dbGetPasswordStmt.QueryRow(username).Scan(&pwd) + if err != nil { + if err == sql.ErrNoRows { + err = fmt.Errorf("user '%s' not known by rivendell", username) + } + return + } + self.password_cache[username] = pwd + } + return } @@ -104,14 +134,18 @@ func (self *Config) Cleanup() { close(self.quit) close(self.done) close(self.getPasswordChan) - //TODO : close db connection + if self.dbh != nil { + self.dbh.Close() + } + if self.dbGetPasswordStmt != nil { + self.dbGetPasswordStmt.Close() + } } func NewConfig(configfile, rdxport_endpoint *string) (conf *Config, err error) { conf = new(Config) conf.configfile = *configfile - err = conf.read_config_file() - if err != nil { + if err = conf.read_config_file(); err != nil { return } conf.quit = make(chan bool) @@ -120,7 +154,9 @@ func NewConfig(configfile, rdxport_endpoint *string) (conf *Config, err error) { conf.password_cache = make(map[string]string) conf.getPasswordChan = make(chan getPasswordRequest) - //TODO : init db connection + if err = conf.init_database(); err != nil { + return + } go conf.dispatchRequests() return -- cgit v0.10.2 From 58f54da16b88f2600c9d53b49ae81e14facd293c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 4 Dec 2015 05:46:53 +0100 Subject: further improved simple web interface diff --git a/importer.go b/importer.go index 8c7e66e..e47bc05 100644 --- a/importer.go +++ b/importer.go @@ -26,7 +26,7 @@ package rhimport import ( // "bytes" - "fmt" + // "fmt" // "io" // "mime/multipart" // "net/http" @@ -36,7 +36,8 @@ import ( type ImportContext struct { Conf *Config UserName string - password string + Password string + Trusted bool GroupName string Cart int Channels int @@ -51,6 +52,8 @@ func NewImportContext(conf *Config, user string, group string, cart int) *Import ctx := new(ImportContext) ctx.Conf = conf ctx.UserName = user + ctx.Password = "" + ctx.Trusted = false ctx.GroupName = group ctx.Cart = cart ctx.Channels = 2 @@ -75,7 +78,7 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { err = res.err return } - ctx.password = res.password + ctx.Password = res.password return } @@ -129,12 +132,14 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { func ImportFile(ctx *ImportContext) (err error) { rhl.Println("ImportFile called for", ctx.SourceFile) - if err = ctx.getPassword(true); err != nil { - return + if ctx.Trusted { + if err = ctx.getPassword(true); err != nil { + return + } } - rhdl.Println("credentials:", ctx.UserName, "/", ctx.password) + rhdl.Printf("credentials: '%s':'%s'", ctx.UserName, ctx.Password) - err = fmt.Errorf("%+v", ctx) +// err = fmt.Errorf("%+v", ctx) return } -- cgit v0.10.2 From a1242a443a5c72ea3b851b8d886a82e0b9d42cf9 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 7 Dec 2015 16:55:35 +0100 Subject: major refactoring of base structure diff --git a/conf.go b/conf.go index 6dcbe84..8e373c0 100644 --- a/conf.go +++ b/conf.go @@ -45,6 +45,7 @@ type getPasswordRequest struct { type Config struct { configfile string RDXportEndpoint string + TempDir string db_host string db_user string db_passwd string @@ -142,7 +143,7 @@ func (self *Config) Cleanup() { } } -func NewConfig(configfile, rdxport_endpoint *string) (conf *Config, err error) { +func NewConfig(configfile, rdxport_endpoint, temp_dir *string) (conf *Config, err error) { conf = new(Config) conf.configfile = *configfile if err = conf.read_config_file(); err != nil { @@ -151,6 +152,7 @@ func NewConfig(configfile, rdxport_endpoint *string) (conf *Config, err error) { conf.quit = make(chan bool) conf.done = make(chan bool) conf.RDXportEndpoint = *rdxport_endpoint + conf.TempDir = *temp_dir conf.password_cache = make(map[string]string) conf.getPasswordChan = make(chan getPasswordRequest) diff --git a/fetcher.go b/fetcher.go new file mode 100644 index 0000000..ead9f5f --- /dev/null +++ b/fetcher.go @@ -0,0 +1,34 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +func FetchFile(ctx *ImportContext) (err error) { + + // TODO: fetch file from ctx.SourceUri and put it into ctx.Conf.TempDir + + ctx.SourceFile = ctx.Conf.TempDir + "/source-file.ogg" + ctx.DeleteSourceFile = true + return +} diff --git a/importer.go b/importer.go index e47bc05..95e3e73 100644 --- a/importer.go +++ b/importer.go @@ -25,12 +25,12 @@ package rhimport import ( - // "bytes" - // "fmt" - // "io" - // "mime/multipart" - // "net/http" - // "os" +// "bytes" +// "fmt" +// "io" +// "mime/multipart" +// "net/http" +// "os" ) type ImportContext struct { @@ -40,24 +40,27 @@ type ImportContext struct { Trusted bool GroupName string Cart int + Cut int Channels int NormalizationLevel int AutotrimLevel int UseMetaData bool + SourceUri string SourceFile string DeleteSourceFile bool } -func NewImportContext(conf *Config, user string, group string, cart int) *ImportContext { +func NewImportContext(conf *Config, user string, group string) *ImportContext { ctx := new(ImportContext) ctx.Conf = conf ctx.UserName = user ctx.Password = "" ctx.Trusted = false ctx.GroupName = group - ctx.Cart = cart + ctx.Cart = 0 + ctx.Cut = 0 ctx.Channels = 2 - ctx.NormalizationLevel = 1200 + ctx.NormalizationLevel = -1200 ctx.AutotrimLevel = 0 ctx.UseMetaData = false ctx.SourceFile = "" @@ -137,9 +140,7 @@ func ImportFile(ctx *ImportContext) (err error) { return } } - rhdl.Printf("credentials: '%s':'%s'", ctx.UserName, ctx.Password) - -// err = fmt.Errorf("%+v", ctx) + rhdl.Printf("%+v", ctx) return } -- cgit v0.10.2 From 02b42d386668926464fd166c6a2dde51246fd9df Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 7 Dec 2015 17:16:40 +0100 Subject: moved DB into seperate module diff --git a/conf.go b/conf.go index 8e373c0..7288a89 100644 --- a/conf.go +++ b/conf.go @@ -25,37 +25,17 @@ package rhimport import ( - "database/sql" - "fmt" - _ "github.com/go-sql-driver/mysql" "github.com/vaughan0/go-ini" ) -type getPasswordResult struct { - password string - err error -} - -type getPasswordRequest struct { - username string - cached bool - response chan getPasswordResult -} - type Config struct { - configfile string - RDXportEndpoint string - TempDir string - db_host string - db_user string - db_passwd string - db_db string - dbh *sql.DB - password_cache map[string]string - getPasswordChan chan getPasswordRequest - dbGetPasswordStmt *sql.Stmt - quit chan bool - done chan bool + configfile string + RDXportEndpoint string + TempDir string + db_host string + db_user string + db_passwd string + db_db string } func get_ini_value(file ini.File, section string, key string, dflt string) string { @@ -79,87 +59,13 @@ func (self *Config) read_config_file() error { return nil } -func (self *Config) init_database() (err error) { - dsn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", self.db_user, self.db_passwd, self.db_host, self.db_db) - if self.dbh, err = sql.Open("mysql", dsn); err != nil { - return - } - if self.dbGetPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?"); err != nil { - return - } - - return -} - -func (self *Config) getPassword(username string, cached bool) (pwd string, err error) { - - if cached { - pwd = self.password_cache[username] - } - - if pwd == "" { - err = self.dbGetPasswordStmt.QueryRow(username).Scan(&pwd) - if err != nil { - if err == sql.ErrNoRows { - err = fmt.Errorf("user '%s' not known by rivendell", username) - } - return - } - self.password_cache[username] = pwd - } - - return -} - -func (self *Config) dispatchRequests() { - defer func() { self.done <- true }() - for { - select { - case <-self.quit: - return - case req := <-self.getPasswordChan: - if req.cached { - rhdl.Println("Config: got getPassword request for", req.username, "(cached)") - } else { - rhdl.Println("Config: got getPassword request for", req.username, "(not cached)") - } - pwd, err := self.getPassword(req.username, req.cached) - req.response <- getPasswordResult{pwd, err} - } - } -} - -func (self *Config) Cleanup() { - self.quit <- true - <-self.done - close(self.quit) - close(self.done) - close(self.getPasswordChan) - if self.dbh != nil { - self.dbh.Close() - } - if self.dbGetPasswordStmt != nil { - self.dbGetPasswordStmt.Close() - } -} - func NewConfig(configfile, rdxport_endpoint, temp_dir *string) (conf *Config, err error) { conf = new(Config) conf.configfile = *configfile if err = conf.read_config_file(); err != nil { return } - conf.quit = make(chan bool) - conf.done = make(chan bool) conf.RDXportEndpoint = *rdxport_endpoint conf.TempDir = *temp_dir - conf.password_cache = make(map[string]string) - conf.getPasswordChan = make(chan getPasswordRequest) - - if err = conf.init_database(); err != nil { - return - } - - go conf.dispatchRequests() return } diff --git a/fetcher.go b/fetcher.go index ead9f5f..1a5801b 100644 --- a/fetcher.go +++ b/fetcher.go @@ -26,9 +26,9 @@ package rhimport func FetchFile(ctx *ImportContext) (err error) { - // TODO: fetch file from ctx.SourceUri and put it into ctx.Conf.TempDir + // TODO: fetch file from ctx.SourceUri and put it into ctx.Config.TempDir - ctx.SourceFile = ctx.Conf.TempDir + "/source-file.ogg" + ctx.SourceFile = ctx.Config.TempDir + "/source-file.ogg" ctx.DeleteSourceFile = true return } diff --git a/importer.go b/importer.go index 95e3e73..5eea054 100644 --- a/importer.go +++ b/importer.go @@ -34,7 +34,8 @@ import ( ) type ImportContext struct { - Conf *Config + *Config + *RDDB UserName string Password string Trusted bool @@ -50,9 +51,10 @@ type ImportContext struct { DeleteSourceFile bool } -func NewImportContext(conf *Config, user string, group string) *ImportContext { +func NewImportContext(conf *Config, rddb *RDDB, user string, group string) *ImportContext { ctx := new(ImportContext) - ctx.Conf = conf + ctx.Config = conf + ctx.RDDB = rddb ctx.UserName = user ctx.Password = "" ctx.Trusted = false @@ -74,7 +76,7 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { req.username = ctx.UserName req.cached = cached req.response = make(chan getPasswordResult) - ctx.Conf.getPasswordChan <- req + ctx.RDDB.getPasswordChan <- req res := <-req.response if res.err != nil { diff --git a/rddb.go b/rddb.go new file mode 100644 index 0000000..2a87388 --- /dev/null +++ b/rddb.go @@ -0,0 +1,131 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "database/sql" + "fmt" + _ "github.com/go-sql-driver/mysql" +) + +type getPasswordResult struct { + password string + err error +} + +type getPasswordRequest struct { + username string + cached bool + response chan getPasswordResult +} + +type RDDB struct { + dbh *sql.DB + password_cache map[string]string + getPasswordChan chan getPasswordRequest + dbGetPasswordStmt *sql.Stmt + quit chan bool + done chan bool +} + +func (self *RDDB) init(conf *Config) (err error) { + dsn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", conf.db_user, conf.db_passwd, conf.db_host, conf.db_db) + if self.dbh, err = sql.Open("mysql", dsn); err != nil { + return + } + if self.dbGetPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?"); err != nil { + return + } + + return +} + +func (self *RDDB) getPassword(username string, cached bool) (pwd string, err error) { + + if cached { + pwd = self.password_cache[username] + } + + if pwd == "" { + err = self.dbGetPasswordStmt.QueryRow(username).Scan(&pwd) + if err != nil { + if err == sql.ErrNoRows { + err = fmt.Errorf("user '%s' not known by rivendell", username) + } + return + } + self.password_cache[username] = pwd + } + + return +} + +func (self *RDDB) dispatchRequests() { + defer func() { self.done <- true }() + for { + select { + case <-self.quit: + return + case req := <-self.getPasswordChan: + if req.cached { + rhdl.Println("RDDB: got getPassword request for", req.username, "(cached)") + } else { + rhdl.Println("RDDB: got getPassword request for", req.username, "(not cached)") + } + pwd, err := self.getPassword(req.username, req.cached) + req.response <- getPasswordResult{pwd, err} + } + } +} + +func (self *RDDB) Cleanup() { + self.quit <- true + <-self.done + close(self.quit) + close(self.done) + close(self.getPasswordChan) + if self.dbh != nil { + self.dbh.Close() + } + if self.dbGetPasswordStmt != nil { + self.dbGetPasswordStmt.Close() + } +} + +func NewRDDB(conf *Config) (db *RDDB, err error) { + db = new(RDDB) + + db.quit = make(chan bool) + db.done = make(chan bool) + db.password_cache = make(map[string]string) + db.getPasswordChan = make(chan getPasswordRequest) + + if err = db.init(conf); err != nil { + return + } + + go db.dispatchRequests() + return +} -- cgit v0.10.2 From 41a24ca6b159072e5b44ecd702551587b03ca34b Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 7 Dec 2015 17:22:51 +0100 Subject: renamed RDDB to RdDb and added code to fetch show infos based on ID diff --git a/importer.go b/importer.go index 5eea054..2b512ed 100644 --- a/importer.go +++ b/importer.go @@ -35,10 +35,11 @@ import ( type ImportContext struct { *Config - *RDDB + *RdDb UserName string Password string Trusted bool + ShowId int GroupName string Cart int Cut int @@ -51,13 +52,14 @@ type ImportContext struct { DeleteSourceFile bool } -func NewImportContext(conf *Config, rddb *RDDB, user string, group string) *ImportContext { +func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *ImportContext { ctx := new(ImportContext) ctx.Config = conf - ctx.RDDB = rddb + ctx.RdDb = rddb ctx.UserName = user ctx.Password = "" ctx.Trusted = false + ctx.ShowId = 0 ctx.GroupName = group ctx.Cart = 0 ctx.Cut = 0 @@ -76,7 +78,7 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { req.username = ctx.UserName req.cached = cached req.response = make(chan getPasswordResult) - ctx.RDDB.getPasswordChan <- req + ctx.RdDb.getPasswordChan <- req res := <-req.response if res.err != nil { @@ -87,6 +89,21 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { return } +func (ctx *ImportContext) getShowInfo() (err error) { + req := getShowInfoRequest{} + req.showid = ctx.ShowId + req.response = make(chan getShowInfoResult) + ctx.RdDb.getShowInfoChan <- req + + res := <-req.response + if res.err != nil { + err = res.err + return + } + rhdl.Printf("Title of show %d is '%s'", ctx.ShowId, res.title) + return +} + // func import_audio(url, file string) (err error) { // var b bytes.Buffer @@ -142,6 +159,11 @@ func ImportFile(ctx *ImportContext) (err error) { return } } + + if err = ctx.getShowInfo(); err != nil { + return + } + rhdl.Printf("%+v", ctx) return diff --git a/rddb.go b/rddb.go index 2a87388..dd1dfec 100644 --- a/rddb.go +++ b/rddb.go @@ -41,35 +41,50 @@ type getPasswordRequest struct { response chan getPasswordResult } -type RDDB struct { - dbh *sql.DB - password_cache map[string]string - getPasswordChan chan getPasswordRequest - dbGetPasswordStmt *sql.Stmt - quit chan bool - done chan bool +type getShowInfoResult struct { + title string + carts map[int]int + err error } -func (self *RDDB) init(conf *Config) (err error) { +type getShowInfoRequest struct { + showid int + response chan getShowInfoResult +} + +type RdDb struct { + dbh *sql.DB + password_cache map[string]string + getPasswordChan chan getPasswordRequest + getPasswordStmt *sql.Stmt + getShowInfoChan chan getShowInfoRequest + getShowInfoStmt *sql.Stmt + quit chan bool + done chan bool +} + +func (self *RdDb) init(conf *Config) (err error) { dsn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", conf.db_user, conf.db_passwd, conf.db_host, conf.db_db) if self.dbh, err = sql.Open("mysql", dsn); err != nil { return } - if self.dbGetPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?"); err != nil { + if self.getPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?;"); err != nil { + return + } + if self.getShowInfoStmt, err = self.dbh.Prepare("select TITLE,MACROS from CART where NUMBER = ?;"); err != nil { return } - return } -func (self *RDDB) getPassword(username string, cached bool) (pwd string, err error) { +func (self *RdDb) getPassword(username string, cached bool) (pwd string, err error) { if cached { pwd = self.password_cache[username] } if pwd == "" { - err = self.dbGetPasswordStmt.QueryRow(username).Scan(&pwd) + err = self.getPasswordStmt.QueryRow(username).Scan(&pwd) if err != nil { if err == sql.ErrNoRows { err = fmt.Errorf("user '%s' not known by rivendell", username) @@ -82,7 +97,23 @@ func (self *RDDB) getPassword(username string, cached bool) (pwd string, err err return } -func (self *RDDB) dispatchRequests() { +func (self *RdDb) getShowInfo(showid int) (title string, carts map[int]int, err error) { + var macros string + err = self.getShowInfoStmt.QueryRow(showid).Scan(&title, ¯os) + if err != nil { + if err == sql.ErrNoRows { + err = fmt.Errorf("show '%d' not found", showid) + } + return + } + + rhdl.Printf("RdDb: macros for showid '%d' are set to '%s'", showid, macros) + // TODO: also fetch cart list + carts = make(map[int]int) + return +} + +func (self *RdDb) dispatchRequests() { defer func() { self.done <- true }() for { select { @@ -90,17 +121,21 @@ func (self *RDDB) dispatchRequests() { return case req := <-self.getPasswordChan: if req.cached { - rhdl.Println("RDDB: got getPassword request for", req.username, "(cached)") + rhdl.Println("RdDb: got getPassword request for", req.username, "(cached)") } else { - rhdl.Println("RDDB: got getPassword request for", req.username, "(not cached)") + rhdl.Println("RdDb: got getPassword request for", req.username, "(not cached)") } pwd, err := self.getPassword(req.username, req.cached) req.response <- getPasswordResult{pwd, err} + case req := <-self.getShowInfoChan: + rhdl.Println("RdDb: got getShowInfo request for", req.showid) + title, carts, err := self.getShowInfo(req.showid) + req.response <- getShowInfoResult{title, carts, err} } } } -func (self *RDDB) Cleanup() { +func (self *RdDb) Cleanup() { self.quit <- true <-self.done close(self.quit) @@ -109,18 +144,22 @@ func (self *RDDB) Cleanup() { if self.dbh != nil { self.dbh.Close() } - if self.dbGetPasswordStmt != nil { - self.dbGetPasswordStmt.Close() + if self.getPasswordStmt != nil { + self.getPasswordStmt.Close() + } + if self.getShowInfoStmt != nil { + self.getPasswordStmt.Close() } } -func NewRDDB(conf *Config) (db *RDDB, err error) { - db = new(RDDB) +func NewRdDb(conf *Config) (db *RdDb, err error) { + db = new(RdDb) db.quit = make(chan bool) db.done = make(chan bool) db.password_cache = make(map[string]string) db.getPasswordChan = make(chan getPasswordRequest) + db.getShowInfoChan = make(chan getShowInfoRequest) if err = db.init(conf); err != nil { return -- cgit v0.10.2 From 9fe963fd9bd3b8be538fbb6b196ec4942feffa7c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 8 Dec 2015 11:50:37 +0100 Subject: import_file should be done diff --git a/fetcher.go b/fetcher.go index 1a5801b..97fe7ff 100644 --- a/fetcher.go +++ b/fetcher.go @@ -24,11 +24,15 @@ package rhimport +import ( + "path" +) + func FetchFile(ctx *ImportContext) (err error) { // TODO: fetch file from ctx.SourceUri and put it into ctx.Config.TempDir - ctx.SourceFile = ctx.Config.TempDir + "/source-file.ogg" + ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(ctx.SourceUri) ctx.DeleteSourceFile = true return } diff --git a/importer.go b/importer.go index 2b512ed..0f8eea8 100644 --- a/importer.go +++ b/importer.go @@ -25,12 +25,17 @@ package rhimport import ( -// "bytes" -// "fmt" -// "io" -// "mime/multipart" -// "net/http" -// "os" + "bytes" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path" +) + +var ( + bool2str = map[bool]string{false: "0", true: "1"} ) type ImportContext struct { @@ -104,52 +109,69 @@ func (ctx *ImportContext) getShowInfo() (err error) { return } -// func import_audio(url, file string) (err error) { - -// var b bytes.Buffer -// w := multipart.NewWriter(&b) - -// if err = w.WriteField("COMMAND", "2"); err != nil { -// return -// } -// if err = w.WriteField("LOGIN_NAME", ""); err != nil { -// return -// } -// if err = w.WriteField("PASSWORD", ""); err != nil { -// return -// } - -// f, err := os.Open(file) -// if err != nil { -// return -// } -// fw, err := w.CreateFormFile("FILENAME", file) -// if err != nil { -// return -// } -// if _, err = io.Copy(fw, f); err != nil { -// return -// } -// f.Close() - -// w.Close() - -// req, err := http.NewRequest("POST", url, &b) -// if err != nil { -// return -// } -// req.Header.Set("Content-Type", w.FormDataContentType()) - -// client := &http.Client{} -// res, err := client.Do(req) -// if err != nil { -// return -// } -// if res.StatusCode != http.StatusOK { -// err = fmt.Errorf("bad status: %s", res.Status) -// } -// return -// } +func import_audio(ctx *ImportContext) (err error) { + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "2"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + if err = w.WriteField("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { + return + } + if err = w.WriteField("CHANNELS", fmt.Sprintf("%d", ctx.Channels)); err != nil { + return + } + if err = w.WriteField("NORMALIZATION_LEVEL", fmt.Sprintf("%d", ctx.NormalizationLevel)); err != nil { + return + } + if err = w.WriteField("AUTOTRIM_LEVEL", fmt.Sprintf("%d", ctx.AutotrimLevel)); err != nil { + return + } + if err = w.WriteField("USE_METADATA", bool2str[ctx.UseMetaData]); err != nil { + return + } + + var f *os.File + var fw io.Writer + if f, err = os.Open(ctx.SourceFile); err != nil { + return + } + if fw, err = w.CreateFormFile("FILENAME", path.Base(ctx.SourceFile)); err != nil { + return + } + if _, err = io.Copy(fw, f); err != nil { + return + } + f.Close() + w.Close() + + var req *http.Request + if req, err = http.NewRequest("POST", ctx.Config.RDXportEndpoint, &b); err != nil { + return + } + req.Header.Set("Content-Type", w.FormDataContentType()) + + client := &http.Client{} + var res *http.Response + if res, err = client.Do(req); err != nil { + return + } + if res.StatusCode != http.StatusOK { + // TODO: better error output + err = fmt.Errorf("bad status: %s", res.Status) + } + return +} func ImportFile(ctx *ImportContext) (err error) { rhl.Println("ImportFile called for", ctx.SourceFile) @@ -160,11 +182,5 @@ func ImportFile(ctx *ImportContext) (err error) { } } - if err = ctx.getShowInfo(); err != nil { - return - } - - rhdl.Printf("%+v", ctx) - - return + return import_audio(ctx) } -- cgit v0.10.2 From 839f5c3b50c4dfcf66cea036bd3d13eec038962c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 8 Dec 2015 12:25:52 +0100 Subject: fetcher parses SourceUri and selects right fetcher function diff --git a/fetcher.go b/fetcher.go index 97fe7ff..26077b8 100644 --- a/fetcher.go +++ b/fetcher.go @@ -25,14 +25,46 @@ package rhimport import ( + "fmt" + "net/url" "path" ) +func FetchFileHttp(ctx *ImportContext, uri *url.URL) (err error) { + rhl.Printf("HTTP/HTTPS fetcher called for '%s'", ctx.SourceUri) + ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path) + ctx.DeleteSourceFile = true + return +} + +func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { + rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) + ctx.SourceFile = uri.Path + ctx.DeleteSourceFile = false + return +} + +type FetchFunc func(*ImportContext, *url.URL) (err error) + +var ( + fetchers = map[string]FetchFunc{ + "http": FetchFileHttp, + "https": FetchFileHttp, + "local": FetchFileLocal, + } +) + func FetchFile(ctx *ImportContext) (err error) { - // TODO: fetch file from ctx.SourceUri and put it into ctx.Config.TempDir + var uri *url.URL + if uri, err = url.Parse(ctx.SourceUri); err != nil { + return + } - ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(ctx.SourceUri) - ctx.DeleteSourceFile = true + if fetcher, ok := fetchers[uri.Scheme]; ok { + err = fetcher(ctx, uri) + } else { + err = fmt.Errorf("No fetcher for uri scheme '%s' found.", uri.Scheme) + } return } -- cgit v0.10.2 From 196f51ef252ec9c79e89e420107e72c844bb8255 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 8 Dec 2015 19:54:46 +0100 Subject: automatic protocol detection for curl based fetcher diff --git a/fetcher.go b/fetcher.go index 26077b8..45bbce4 100644 --- a/fetcher.go +++ b/fetcher.go @@ -26,14 +26,44 @@ package rhimport import ( "fmt" + "github.com/golang-basic/go-curl" "net/url" "path" + "time" ) -func FetchFileHttp(ctx *ImportContext, uri *url.URL) (err error) { - rhl.Printf("HTTP/HTTPS fetcher called for '%s'", ctx.SourceUri) - ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path) - ctx.DeleteSourceFile = true +func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { + rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) + + easy := curl.EasyInit() + defer easy.Cleanup() + + if easy != nil { + ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path) + + easy.Setopt(curl.OPT_URL, ctx.SourceUri) + easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool { + // TODO: actually store data to ctx.SourceFile + return true + }) + + easy.Setopt(curl.OPT_NOPROGRESS, false) + started := int64(0) + easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { + if started == 0 { + started = time.Now().Unix() + } + fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started))) + return true + }) + + if err = easy.Perform(); err != nil { + return + } + fmt.Printf("\n") + + ctx.DeleteSourceFile = true + } return } @@ -48,12 +78,35 @@ type FetchFunc func(*ImportContext, *url.URL) (err error) var ( fetchers = map[string]FetchFunc{ - "http": FetchFileHttp, - "https": FetchFileHttp, "local": FetchFileLocal, } + curl_protos = map[string]bool{ + "http": false, "https": false, + "ftp": false, "ftps": false, + } ) +func init() { + curl.GlobalInit(curl.GLOBAL_ALL) + + info := curl.VersionInfo(curl.VERSION_FIRST) + protos := info.Protocols + for _, proto := range protos { + if _, ok := curl_protos[proto]; ok { + rhdl.Printf("curl: enabling protocol %s", proto) + fetchers[proto] = FetchFileCurl + curl_protos[proto] = true + } else { + rhdl.Printf("curl: ignoring protocol %s", proto) + } + } + for proto, enabled := range curl_protos { + if !enabled { + rhl.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto) + } + } +} + func FetchFile(ctx *ImportContext) (err error) { var uri *url.URL -- cgit v0.10.2 From 24f401a03888c0d2456b472f55866269ce121007 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 9 Dec 2015 06:31:55 +0100 Subject: first version of curl base fetcher works now diff --git a/fetcher.go b/fetcher.go index 45bbce4..1770343 100644 --- a/fetcher.go +++ b/fetcher.go @@ -27,11 +27,61 @@ package rhimport import ( "fmt" "github.com/golang-basic/go-curl" + "mime" "net/url" + "os" "path" + "strings" "time" ) +type CurlCBData struct { + basepath string + filename string + remotename string + *os.File +} + +func (self *CurlCBData) Cleanup() { + if self.File != nil { + self.File.Close() + } +} + +func curlHeaderCallback(ptr []byte, userdata interface{}) bool { + hdr := fmt.Sprintf("%s", ptr) + data := userdata.(*CurlCBData) + + if strings.HasPrefix(hdr, "Content-Disposition:") { + if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil { + if mediatype == "attachment" { + data.filename = data.basepath + "/" + params["filename"] + } + } + } + return true +} + +func curlWriteCallback(ptr []byte, userdata interface{}) bool { + data := userdata.(*CurlCBData) + if data.File == nil { + if data.filename == "" { + data.filename = data.basepath + "/" + data.remotename + } + fp, err := os.OpenFile(data.filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + rhl.Printf("Unable to create file %s: %s", data.filename, err) + return false + } + data.File = fp + } + if _, err := data.File.Write(ptr); err != nil { + rhl.Printf("Unable to write file %s: %s", data.filename, err) + return false + } + return true +} + func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) @@ -39,13 +89,19 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { defer easy.Cleanup() if easy != nil { - ctx.SourceFile = ctx.Config.TempDir + "/" + path.Base(uri.Path) + easy.Setopt(curl.OPT_FOLLOWLOCATION, true) easy.Setopt(curl.OPT_URL, ctx.SourceUri) - easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool { - // TODO: actually store data to ctx.SourceFile - return true - }) + + cbdata := &CurlCBData{remotename: path.Base(uri.Path)} + defer cbdata.Cleanup() + cbdata.basepath = ctx.Config.TempDir // TODO: create temporary directory + + easy.Setopt(curl.OPT_HEADERFUNCTION, curlHeaderCallback) + easy.Setopt(curl.OPT_HEADERDATA, cbdata) + + easy.Setopt(curl.OPT_WRITEFUNCTION, curlWriteCallback) + easy.Setopt(curl.OPT_WRITEDATA, cbdata) easy.Setopt(curl.OPT_NOPROGRESS, false) started := int64(0) @@ -56,12 +112,14 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started))) return true }) + easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { return } fmt.Printf("\n") + ctx.SourceFile = cbdata.filename ctx.DeleteSourceFile = true } return -- cgit v0.10.2 From db0d524595106b1e907df04baa3da8eb17527f6d Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 10 Dec 2015 15:56:01 +0100 Subject: importer deletes source file and dir if requested diff --git a/fetcher.go b/fetcher.go index 1770343..b4e2f0c 100644 --- a/fetcher.go +++ b/fetcher.go @@ -27,6 +27,7 @@ package rhimport import ( "fmt" "github.com/golang-basic/go-curl" + "io/ioutil" "mime" "net/url" "os" @@ -95,7 +96,9 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { cbdata := &CurlCBData{remotename: path.Base(uri.Path)} defer cbdata.Cleanup() - cbdata.basepath = ctx.Config.TempDir // TODO: create temporary directory + if cbdata.basepath, err = ioutil.TempDir(ctx.Config.TempDir, "rhimportd-"); err != nil { + return + } easy.Setopt(curl.OPT_HEADERFUNCTION, curlHeaderCallback) easy.Setopt(curl.OPT_HEADERDATA, cbdata) @@ -121,6 +124,7 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { ctx.SourceFile = cbdata.filename ctx.DeleteSourceFile = true + ctx.DeleteSourceDir = true } return } @@ -129,6 +133,7 @@ func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) ctx.SourceFile = uri.Path ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false return } diff --git a/importer.go b/importer.go index 0f8eea8..0d0a836 100644 --- a/importer.go +++ b/importer.go @@ -55,6 +55,7 @@ type ImportContext struct { SourceUri string SourceFile string DeleteSourceFile bool + DeleteSourceDir bool } func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *ImportContext { @@ -74,6 +75,7 @@ func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *Impo ctx.UseMetaData = false ctx.SourceFile = "" ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false return ctx } @@ -173,8 +175,27 @@ func import_audio(ctx *ImportContext) (err error) { return } +func cleanup_files(ctx *ImportContext) { + if(ctx.DeleteSourceFile) { + rhdl.Printf("importer: removing file: %s", ctx.SourceFile) + if err := os.Remove(ctx.SourceFile); err != nil { + rhl.Printf("importer: error removing source file: %s", err) + return + } + if(ctx.DeleteSourceDir) { + dir := path.Dir(ctx.SourceFile) + rhdl.Printf("importer: also removing directory: %s", dir) + if err := os.Remove(dir); err != nil { + rhl.Printf("importer: error removing source directory: %s", err) + } + } + } + return +} + func ImportFile(ctx *ImportContext) (err error) { rhl.Println("ImportFile called for", ctx.SourceFile) + defer cleanup_files(ctx) if ctx.Trusted { if err = ctx.getPassword(true); err != nil { @@ -182,5 +203,9 @@ func ImportFile(ctx *ImportContext) (err error) { } } - return import_audio(ctx) + if err = import_audio(ctx); err != nil { + return + } + + return } -- cgit v0.10.2 From 4b0cb427306cb5bab60e6abdd16dbd7bd8df2a85 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 10 Dec 2015 17:30:40 +0100 Subject: added support for progress callbacks diff --git a/fetcher.go b/fetcher.go index b4e2f0c..03cf3da 100644 --- a/fetcher.go +++ b/fetcher.go @@ -33,7 +33,6 @@ import ( "os" "path" "strings" - "time" ) type CurlCBData struct { @@ -107,12 +106,11 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { easy.Setopt(curl.OPT_WRITEDATA, cbdata) easy.Setopt(curl.OPT_NOPROGRESS, false) - started := int64(0) easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { - if started == 0 { - started = time.Now().Unix() + ctx := userdata.(*ImportContext) + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(1, "downloading", dlnow/dltotal, ctx.ProgressCallBackData) } - fmt.Printf("Downloaded: %3.2f%%, Speed: %.1fKiB/s \r", dlnow/dltotal*100, dlnow/1000/float64((time.Now().Unix()-started))) return true }) easy.Setopt(curl.OPT_PROGRESSDATA, ctx) @@ -131,6 +129,9 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData) + } ctx.SourceFile = uri.Path ctx.DeleteSourceFile = false ctx.DeleteSourceDir = false diff --git a/importer.go b/importer.go index 0d0a836..68b71a6 100644 --- a/importer.go +++ b/importer.go @@ -41,21 +41,23 @@ var ( type ImportContext struct { *Config *RdDb - UserName string - Password string - Trusted bool - ShowId int - GroupName string - Cart int - Cut int - Channels int - NormalizationLevel int - AutotrimLevel int - UseMetaData bool - SourceUri string - SourceFile string - DeleteSourceFile bool - DeleteSourceDir bool + UserName string + Password string + Trusted bool + ShowId int + GroupName string + Cart int + Cut int + Channels int + NormalizationLevel int + AutotrimLevel int + UseMetaData bool + SourceUri string + SourceFile string + DeleteSourceFile bool + DeleteSourceDir bool + ProgressCallBack func(step int, step_name string, progress float64, userdata interface{}) + ProgressCallBackData interface{} } func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *ImportContext { @@ -76,6 +78,7 @@ func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *Impo ctx.SourceFile = "" ctx.DeleteSourceFile = false ctx.DeleteSourceDir = false + ctx.ProgressCallBack = nil return ctx } @@ -168,6 +171,8 @@ func import_audio(ctx *ImportContext) (err error) { if res, err = client.Do(req); err != nil { return } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { // TODO: better error output err = fmt.Errorf("bad status: %s", res.Status) @@ -176,13 +181,13 @@ func import_audio(ctx *ImportContext) (err error) { } func cleanup_files(ctx *ImportContext) { - if(ctx.DeleteSourceFile) { + if ctx.DeleteSourceFile { rhdl.Printf("importer: removing file: %s", ctx.SourceFile) if err := os.Remove(ctx.SourceFile); err != nil { rhl.Printf("importer: error removing source file: %s", err) return } - if(ctx.DeleteSourceDir) { + if ctx.DeleteSourceDir { dir := path.Dir(ctx.SourceFile) rhdl.Printf("importer: also removing directory: %s", dir) if err := os.Remove(dir); err != nil { @@ -197,6 +202,10 @@ func ImportFile(ctx *ImportContext) (err error) { rhl.Println("ImportFile called for", ctx.SourceFile) defer cleanup_files(ctx) + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(2, "importing", 0.0, ctx.ProgressCallBackData) + } + if ctx.Trusted { if err = ctx.getPassword(true); err != nil { return @@ -207,5 +216,10 @@ func ImportFile(ctx *ImportContext) (err error) { return } + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(2, "importing", 1.0, ctx.ProgressCallBackData) + } + + rhl.Println("ImportFile succesfully imported", ctx.SourceFile) return } -- cgit v0.10.2 From 3a2c7075ddd45e792e50fa45f1c2fd69c73fbe53 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 10 Dec 2015 17:59:30 +0100 Subject: added add_cart function diff --git a/importer.go b/importer.go index 68b71a6..c7b1a66 100644 --- a/importer.go +++ b/importer.go @@ -114,6 +114,57 @@ func (ctx *ImportContext) getShowInfo() (err error) { return } +func send_post_request(url string, b *bytes.Buffer, contenttype string) (err error) { + var req *http.Request + if req, err = http.NewRequest("POST", url, b); err != nil { + return + } + if contenttype != "" { + req.Header.Set("Content-Type", contenttype) + } + + client := &http.Client{} + var res *http.Response + if res, err = client.Do(req); err != nil { + return + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + // TODO: better error output + err = fmt.Errorf("bad status: %s", res.Status) + } + return +} + +func add_cart(ctx *ImportContext) (err error) { + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "12"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("GRPOUP_NAME", ctx.GroupName); err != nil { + return + } + if err = w.WriteField("TYPE", "audio"); err != nil { + return + } + if ctx.Cart != 0 { + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + } + w.Close() + return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) +} + func import_audio(ctx *ImportContext) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -160,24 +211,7 @@ func import_audio(ctx *ImportContext) (err error) { f.Close() w.Close() - var req *http.Request - if req, err = http.NewRequest("POST", ctx.Config.RDXportEndpoint, &b); err != nil { - return - } - req.Header.Set("Content-Type", w.FormDataContentType()) - - client := &http.Client{} - var res *http.Response - if res, err = client.Do(req); err != nil { - return - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - // TODO: better error output - err = fmt.Errorf("bad status: %s", res.Status) - } - return + return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) } func cleanup_files(ctx *ImportContext) { -- cgit v0.10.2 From 4254d06d1fc062fc0d62ee403e8ac3ce0157888f Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 11 Dec 2015 02:36:36 +0100 Subject: minor code refactoring diff --git a/core.go b/core.go new file mode 100644 index 0000000..5498499 --- /dev/null +++ b/core.go @@ -0,0 +1,43 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + // "io/ioutil" + "github.com/golang-basic/go-curl" + "log" + "os" +) + +var ( + rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) + rhdl = log.New(os.Stderr, "[rhimport-dbg]\t", log.LstdFlags) + //rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) +) + +func init() { + curl.GlobalInit(curl.GLOBAL_ALL) + fetcher_init() +} diff --git a/fetcher.go b/fetcher.go index 03cf3da..ccbca43 100644 --- a/fetcher.go +++ b/fetcher.go @@ -150,9 +150,7 @@ var ( } ) -func init() { - curl.GlobalInit(curl.GLOBAL_ALL) - +func fetcher_init() { info := curl.VersionInfo(curl.VERSION_FIRST) protos := info.Protocols for _, proto := range protos { diff --git a/log.go b/log.go deleted file mode 100644 index b192c30..0000000 --- a/log.go +++ /dev/null @@ -1,37 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - // "io/ioutil" - "log" - "os" -) - -var ( - rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) - rhdl = log.New(os.Stderr, "[rhimport-dbg]\t", log.LstdFlags) - //rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) -) -- cgit v0.10.2 From cf1a20001d92211d960965dedadc45f264f2b27a Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 11 Dec 2015 02:48:57 +0100 Subject: added RemovCart and AddCut commands diff --git a/importer.go b/importer.go index c7b1a66..8995717 100644 --- a/importer.go +++ b/importer.go @@ -137,6 +137,7 @@ func send_post_request(url string, b *bytes.Buffer, contenttype string) (err err return } + func add_cart(ctx *ImportContext) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -165,6 +166,47 @@ func add_cart(ctx *ImportContext) (err error) { return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) } +func add_cut(ctx *ImportContext) (err error) { + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "10"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + w.Close() + return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) +} + +func remove_cart(ctx *ImportContext) (err error) { + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "13"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + w.Close() + return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) +} + + func import_audio(ctx *ImportContext) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) -- cgit v0.10.2 From fdae25e68b9564b59441c3a3a0fd1b65e54740a7 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 11 Dec 2015 03:57:49 +0100 Subject: curl based importer diff --git a/fetcher.go b/fetcher.go index ccbca43..05074b1 100644 --- a/fetcher.go +++ b/fetcher.go @@ -35,14 +35,14 @@ import ( "strings" ) -type CurlCBData struct { +type FetcherCurlCBData struct { basepath string filename string remotename string *os.File } -func (self *CurlCBData) Cleanup() { +func (self *FetcherCurlCBData) Cleanup() { if self.File != nil { self.File.Close() } @@ -50,7 +50,7 @@ func (self *CurlCBData) Cleanup() { func curlHeaderCallback(ptr []byte, userdata interface{}) bool { hdr := fmt.Sprintf("%s", ptr) - data := userdata.(*CurlCBData) + data := userdata.(*FetcherCurlCBData) if strings.HasPrefix(hdr, "Content-Disposition:") { if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil { @@ -63,7 +63,7 @@ func curlHeaderCallback(ptr []byte, userdata interface{}) bool { } func curlWriteCallback(ptr []byte, userdata interface{}) bool { - data := userdata.(*CurlCBData) + data := userdata.(*FetcherCurlCBData) if data.File == nil { if data.filename == "" { data.filename = data.basepath + "/" + data.remotename @@ -86,14 +86,13 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) easy := curl.EasyInit() - defer easy.Cleanup() - if easy != nil { + defer easy.Cleanup() easy.Setopt(curl.OPT_FOLLOWLOCATION, true) easy.Setopt(curl.OPT_URL, ctx.SourceUri) - cbdata := &CurlCBData{remotename: path.Base(uri.Path)} + cbdata := &FetcherCurlCBData{remotename: path.Base(uri.Path)} defer cbdata.Cleanup() if cbdata.basepath, err = ioutil.TempDir(ctx.Config.TempDir, "rhimportd-"); err != nil { return @@ -118,12 +117,14 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { if err = easy.Perform(); err != nil { return } - fmt.Printf("\n") ctx.SourceFile = cbdata.filename ctx.DeleteSourceFile = true ctx.DeleteSourceDir = true + } else { + err = fmt.Errorf("Error initializing libcurl") } + return } diff --git a/importer.go b/importer.go index 8995717..bbe4874 100644 --- a/importer.go +++ b/importer.go @@ -27,6 +27,7 @@ package rhimport import ( "bytes" "fmt" + "github.com/golang-basic/go-curl" "io" "mime/multipart" "net/http" @@ -114,30 +115,6 @@ func (ctx *ImportContext) getShowInfo() (err error) { return } -func send_post_request(url string, b *bytes.Buffer, contenttype string) (err error) { - var req *http.Request - if req, err = http.NewRequest("POST", url, b); err != nil { - return - } - if contenttype != "" { - req.Header.Set("Content-Type", contenttype) - } - - client := &http.Client{} - var res *http.Response - if res, err = client.Do(req); err != nil { - return - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - // TODO: better error output - err = fmt.Errorf("bad status: %s", res.Status) - } - return -} - - func add_cart(ctx *ImportContext) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -206,54 +183,113 @@ func remove_cart(ctx *ImportContext) (err error) { return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) } - -func import_audio(ctx *ImportContext) (err error) { - var b bytes.Buffer - w := multipart.NewWriter(&b) - - if err = w.WriteField("COMMAND", "2"); err != nil { +func send_post_request(url string, b *bytes.Buffer, contenttype string) (err error) { + var req *http.Request + if req, err = http.NewRequest("POST", url, b); err != nil { return } - if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + if contenttype != "" { + req.Header.Set("Content-Type", contenttype) + } + + client := &http.Client{} + var res *http.Response + if res, err = client.Do(req); err != nil { return } - if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + defer res.Body.Close() + + fmt.Printf("Response Code: %d\nBody:\n", res.StatusCode) + io.Copy(os.Stdout, res.Body) + return +} + +func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *curl.Form, err error) { + form = curl.NewForm() + + if err = form.Add("COMMAND", "2"); err != nil { return } - if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + if err = form.Add("LOGIN_NAME", ctx.UserName); err != nil { return } - if err = w.WriteField("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { + if err = form.Add("PASSWORD", ctx.Password); err != nil { return } - if err = w.WriteField("CHANNELS", fmt.Sprintf("%d", ctx.Channels)); err != nil { + if err = form.Add("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { return } - if err = w.WriteField("NORMALIZATION_LEVEL", fmt.Sprintf("%d", ctx.NormalizationLevel)); err != nil { + if err = form.Add("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { return } - if err = w.WriteField("AUTOTRIM_LEVEL", fmt.Sprintf("%d", ctx.AutotrimLevel)); err != nil { + if err = form.Add("CHANNELS", fmt.Sprintf("%d", ctx.Channels)); err != nil { return } - if err = w.WriteField("USE_METADATA", bool2str[ctx.UseMetaData]); err != nil { + if err = form.Add("NORMALIZATION_LEVEL", fmt.Sprintf("%d", ctx.NormalizationLevel)); err != nil { return } - - var f *os.File - var fw io.Writer - if f, err = os.Open(ctx.SourceFile); err != nil { + if err = form.Add("AUTOTRIM_LEVEL", fmt.Sprintf("%d", ctx.AutotrimLevel)); err != nil { return } - if fw, err = w.CreateFormFile("FILENAME", path.Base(ctx.SourceFile)); err != nil { + if err = form.Add("USE_METADATA", bool2str[ctx.UseMetaData]); err != nil { return } - if _, err = io.Copy(fw, f); err != nil { + if err = form.AddFile("FILENAME", ctx.SourceFile); err != nil { return } - f.Close() - w.Close() - return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) + return +} + +func import_audio(ctx *ImportContext) (err error) { + easy := curl.EasyInit() + + if easy != nil { + defer easy.Cleanup() + + easy.Setopt(curl.OPT_URL, ctx.Config.RDXportEndpoint) + easy.Setopt(curl.OPT_POST, true) + + var form *curl.Form + if form, err = import_audio_create_request(ctx, easy); err != nil { + return + } + easy.Setopt(curl.OPT_HTTPPOST, form) + easy.Setopt(curl.OPT_HTTPHEADER, []string{"Expect:"}) + + var resbody bytes.Buffer + easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool { + b := userdata.(*bytes.Buffer) + b.Write(ptr) + return true + }) + easy.Setopt(curl.OPT_WRITEDATA, &resbody) + + easy.Setopt(curl.OPT_NOPROGRESS, false) + easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { + ctx := userdata.(*ImportContext) + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(2, "importing", ulnow/ultotal, ctx.ProgressCallBackData) + } + return true + }) + easy.Setopt(curl.OPT_PROGRESSDATA, ctx) + + if err = easy.Perform(); err != nil { + return + } + + var status_code interface{} + if status_code, err = easy.Getinfo(curl.INFO_RESPONSE_CODE); err != nil { + return + } + fmt.Printf("Response Code: %d\nBody:\n", status_code.(int)) + resbody.WriteTo(os.Stdout) + } else { + err = fmt.Errorf("Error initializing libcurl") + } + + return } func cleanup_files(ctx *ImportContext) { -- cgit v0.10.2 From ab450c42afb8df8b1042272cfa0a693b9461af4d Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 11 Dec 2015 05:11:32 +0100 Subject: parsing RDWebResult works now diff --git a/importer.go b/importer.go index bbe4874..55fbe62 100644 --- a/importer.go +++ b/importer.go @@ -25,7 +25,9 @@ package rhimport import ( + "bufio" "bytes" + "encoding/xml" "fmt" "github.com/golang-basic/go-curl" "io" @@ -115,7 +117,23 @@ func (ctx *ImportContext) getShowInfo() (err error) { return } -func add_cart(ctx *ImportContext) (err error) { +type RDWebResult struct { + ResponseCode int `xml:"ResponseCode"` + ErrorString string `xml:"ErrorString"` + AudioConvertError string `xml:"AudioConvertError"` +} + +func decodeRDWebResult(data io.Reader) (result *RDWebResult, err error) { + decoder := xml.NewDecoder(data) + result = &RDWebResult{} + if xmlerr := decoder.Decode(result); xmlerr != nil { + err = fmt.Errorf("Error parsing XML response: %s", xmlerr) + return + } + return +} + +func add_cart(ctx *ImportContext) (result *RDWebResult, err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -141,9 +159,10 @@ func add_cart(ctx *ImportContext) (err error) { } w.Close() return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) + // TODO: on success this returns ... } -func add_cut(ctx *ImportContext) (err error) { +func add_cut(ctx *ImportContext) (result *RDWebResult, err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -161,9 +180,10 @@ func add_cut(ctx *ImportContext) (err error) { } w.Close() return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) + // TODO: on success this returns ... } -func remove_cart(ctx *ImportContext) (err error) { +func remove_cart(ctx *ImportContext) (result *RDWebResult, err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -183,7 +203,7 @@ func remove_cart(ctx *ImportContext) (err error) { return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) } -func send_post_request(url string, b *bytes.Buffer, contenttype string) (err error) { +func send_post_request(url string, b *bytes.Buffer, contenttype string) (result *RDWebResult, err error) { var req *http.Request if req, err = http.NewRequest("POST", url, b); err != nil { return @@ -199,9 +219,8 @@ func send_post_request(url string, b *bytes.Buffer, contenttype string) (err err } defer res.Body.Close() - fmt.Printf("Response Code: %d\nBody:\n", res.StatusCode) - io.Copy(os.Stdout, res.Body) - return + // TODO: some commands might return something else... + return decodeRDWebResult(res.Body) } func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *curl.Form, err error) { @@ -241,7 +260,7 @@ func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *cur return } -func import_audio(ctx *ImportContext) (err error) { +func import_audio(ctx *ImportContext) (result *RDWebResult, err error) { easy := curl.EasyInit() if easy != nil { @@ -279,12 +298,11 @@ func import_audio(ctx *ImportContext) (err error) { return } - var status_code interface{} - if status_code, err = easy.Getinfo(curl.INFO_RESPONSE_CODE); err != nil { - return - } - fmt.Printf("Response Code: %d\nBody:\n", status_code.(int)) - resbody.WriteTo(os.Stdout) + // var status_code interface{} + // if status_code, err = easy.Getinfo(curl.INFO_RESPONSE_CODE); err != nil { + // return + // } + return decodeRDWebResult(bufio.NewReader(&resbody)) } else { err = fmt.Errorf("Error initializing libcurl") } @@ -324,14 +342,16 @@ func ImportFile(ctx *ImportContext) (err error) { } } - if err = import_audio(ctx); err != nil { + var result *RDWebResult + if result, err = import_audio(ctx); err != nil { return } - if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(2, "importing", 1.0, ctx.ProgressCallBackData) } + rhdl.Printf("StatusCode: %d, ErrorString: '%s', AudioConverterError: %s\n", result.ResponseCode, result.ErrorString, result.AudioConvertError) + rhl.Println("ImportFile succesfully imported", ctx.SourceFile) return } -- cgit v0.10.2 From d6dd3675bfe43d0fda64b52eec81efbcd7def828 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 12 Dec 2015 02:41:25 +0100 Subject: ImportFile now passes through result from RDXport diff --git a/importer.go b/importer.go index 55fbe62..e29e4df 100644 --- a/importer.go +++ b/importer.go @@ -117,16 +117,33 @@ func (ctx *ImportContext) getShowInfo() (err error) { return } +type ImportResult struct { + ResponseCode int + ErrorString string + Cart int + Cut int +} + +func NewImportResult(rdres *RDWebResult) *ImportResult { + res := new(ImportResult) + res.ResponseCode = rdres.ResponseCode + res.ErrorString = rdres.ErrorString + if rdres.AudioConvertError != 0 { + res.ErrorString += fmt.Sprint(", Audio Convert Error: %d", rdres.AudioConvertError) + } + return res +} + type RDWebResult struct { ResponseCode int `xml:"ResponseCode"` ErrorString string `xml:"ErrorString"` - AudioConvertError string `xml:"AudioConvertError"` + AudioConvertError int `xml:"AudioConvertError"` } -func decodeRDWebResult(data io.Reader) (result *RDWebResult, err error) { +func NewRDWebResultFromXML(data io.Reader) (res *RDWebResult, err error) { decoder := xml.NewDecoder(data) - result = &RDWebResult{} - if xmlerr := decoder.Decode(result); xmlerr != nil { + res = &RDWebResult{} + if xmlerr := decoder.Decode(res); xmlerr != nil { err = fmt.Errorf("Error parsing XML response: %s", xmlerr) return } @@ -220,7 +237,7 @@ func send_post_request(url string, b *bytes.Buffer, contenttype string) (result defer res.Body.Close() // TODO: some commands might return something else... - return decodeRDWebResult(res.Body) + return NewRDWebResultFromXML(res.Body) } func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *curl.Form, err error) { @@ -302,7 +319,7 @@ func import_audio(ctx *ImportContext) (result *RDWebResult, err error) { // if status_code, err = easy.Getinfo(curl.INFO_RESPONSE_CODE); err != nil { // return // } - return decodeRDWebResult(bufio.NewReader(&resbody)) + return NewRDWebResultFromXML(bufio.NewReader(&resbody)) } else { err = fmt.Errorf("Error initializing libcurl") } @@ -328,7 +345,7 @@ func cleanup_files(ctx *ImportContext) { return } -func ImportFile(ctx *ImportContext) (err error) { +func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { rhl.Println("ImportFile called for", ctx.SourceFile) defer cleanup_files(ctx) @@ -342,16 +359,21 @@ func ImportFile(ctx *ImportContext) (err error) { } } - var result *RDWebResult - if result, err = import_audio(ctx); err != nil { + var rdres *RDWebResult + if rdres, err = import_audio(ctx); err != nil { return } if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(2, "importing", 1.0, ctx.ProgressCallBackData) } - rhdl.Printf("StatusCode: %d, ErrorString: '%s', AudioConverterError: %s\n", result.ResponseCode, result.ErrorString, result.AudioConvertError) + res = NewImportResult(rdres) + rhdl.Printf("StatusCode: %d, ErrorString: '%s'\n", res.ResponseCode, res.ErrorString) - rhl.Println("ImportFile succesfully imported", ctx.SourceFile) + if res.ResponseCode == http.StatusOK { + rhl.Println("ImportFile succesfully imported", ctx.SourceFile) + } else { + rhl.Println("ImportFile import of", ctx.SourceFile, "was unsuccesful") + } return } -- cgit v0.10.2 From a8a8bc8ba1ef4a04005587430c1e61a1c1d325f3 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 12 Dec 2015 03:37:52 +0100 Subject: added xml parse for response of rdxport diff --git a/importer.go b/importer.go index e29e4df..749c895 100644 --- a/importer.go +++ b/importer.go @@ -27,10 +27,8 @@ package rhimport import ( "bufio" "bytes" - "encoding/xml" "fmt" "github.com/golang-basic/go-curl" - "io" "mime/multipart" "net/http" "os" @@ -134,22 +132,6 @@ func NewImportResult(rdres *RDWebResult) *ImportResult { return res } -type RDWebResult struct { - ResponseCode int `xml:"ResponseCode"` - ErrorString string `xml:"ErrorString"` - AudioConvertError int `xml:"AudioConvertError"` -} - -func NewRDWebResultFromXML(data io.Reader) (res *RDWebResult, err error) { - decoder := xml.NewDecoder(data) - res = &RDWebResult{} - if xmlerr := decoder.Decode(res); xmlerr != nil { - err = fmt.Errorf("Error parsing XML response: %s", xmlerr) - return - } - return -} - func add_cart(ctx *ImportContext) (result *RDWebResult, err error) { var b bytes.Buffer w := multipart.NewWriter(&b) diff --git a/rdxport_responses.go b/rdxport_responses.go new file mode 100644 index 0000000..db309d9 --- /dev/null +++ b/rdxport_responses.go @@ -0,0 +1,150 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "encoding/xml" + "fmt" + "io" +) + +type RDWebResult struct { + ResponseCode int `xml:"ResponseCode"` + ErrorString string `xml:"ErrorString"` + AudioConvertError int `xml:"AudioConvertError"` +} + +func NewRDWebResultFromXML(data io.Reader) (res *RDWebResult, err error) { + decoder := xml.NewDecoder(data) + res = &RDWebResult{} + if xmlerr := decoder.Decode(res); xmlerr != nil { + err = fmt.Errorf("Error parsing XML response: %s", xmlerr) + return + } + return +} + +type RDCartAdd struct { + Carts []RDCart `xml:"cart"` +} + +type RDCart struct { + Number int `xml:"number"` + Type string `xml:"type"` + GroupName string `xml:"groupName"` + Title string `xml:"title"` + Artist string `xml:"artist"` + Album string `xml:"album"` + Year string `xml:"year"` + Label string `xml:"label"` + Client string `xml:"client"` + Agency string `xml:"agency"` + Publisher string `xml:"publisher"` + Composer string `xml:"composer"` + UserDefined string `xml:"userDefined"` + UsageCode int `xml:"usageCode"` + ForcedLength string `xml:"forcedLength"` + AverageLength string `xml:"averageLength"` + LengthDeviation string `xml:"lengthDeviation"` + AverageSegueLength string `xml:"averageSegueLenth"` + AverageHookLength string `xml:"averageHookLength"` + CutQuantity int `xml:"cutQuantity"` + LastCutPlayer int `xml:"lastCutPlayed"` + Validity int `xml:"validity"` + EnforceLength bool `xml:"enforceLength"` + Asynchronous bool `xml:"asyncronous"` + Owner string `xml:"owner"` + MetadataDatetime string `xml:"metadataDatetime"` +} + +func NewRDCartAddFromXML(data io.Reader) (cart *RDCartAdd, err error) { + decoder := xml.NewDecoder(data) + cart = &RDCartAdd{} + if xmlerr := decoder.Decode(cart); xmlerr != nil { + err = fmt.Errorf("Error parsing XML response: %s", xmlerr) + return + } + return +} + +type RDCutAdd struct { + Cuts []RDCut `xml:"cut"` +} + +type RDCut struct { + Name string `xml:"cutName"` + CartNumber int `xml:"cartNumber"` + Number int `xml:"cutNumber"` + EverGreen bool `xml:"evergreen"` + Description string `xml:"description"` + OutCue string `xml:"outcue"` + ISRC string `xml:"isrc"` + ISCI string `xml:"isci"` + Length int `xml:"length"` + OriginDateTime string `xml:"originDatetime"` + StartDateTime string `xml:"startDatetime"` + EndDateTime string `xml:"endDatetime"` + Sunday bool `xml:"sun"` + Monday bool `xml:"mon"` + Tuesday bool `xml:"tue"` + Wednesday bool `xml:"wed"` + Thursday bool `xml:"thu"` + Friday bool `xml:"fri"` + Saturday bool `xml:"sat"` + StartDaypart string `xml:"startDaypart"` + EndDayPart string `xml:"endDaypart"` + OriginName string `xml:"originName"` + Weight int `xml:"weight"` + LastPlayDateTime string `xml:"lastPlayDatetime"` + PlayCounter int `xml:"playCounter"` + LocalCounter int `xml:"localCounter"` + Validiy int `xml:"validity"` + CondingFormat int `xml:"codingFormat"` + SampleRate int `xml:"sampleRate"` + BitRate int `xml:"bitRate"` + Channels int `xml:"channels"` + PlayGain int `xml:"playGain"` + StartPoint int `xml:"startPoint"` + EndPoint int `xml:"endPoint"` + FadeUpPoint int `xml:"fadeupPoint"` + FadeDownPoint int `xml:"fadedownPoint"` + SegueStartPoint int `xml:"segueStartPoint"` + SegueEndPoint int `xml:"segueEndPoint"` + SegueGain int `xml:"segueGain"` + HookStartPoint int `xml:"hookStartPoint"` + HookEndPoint int `xml:"hookEndPoint"` + TalkStartPoint int `xml:"talkStartPoint"` + TalkEndPoint int `xml:"talkEndPoint"` +} + +func NewRDCutAddFromXML(data io.Reader) (cut *RDCutAdd, err error) { + decoder := xml.NewDecoder(data) + cut = &RDCutAdd{} + if xmlerr := decoder.Decode(cut); xmlerr != nil { + err = fmt.Errorf("Error parsing XML response: %s", xmlerr) + return + } + return +} -- cgit v0.10.2 From 771c4b7542a8f13aa723e1836478f01154612086 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 12 Dec 2015 03:58:55 +0100 Subject: add_cart and add_cut now correctly parse the response diff --git a/importer.go b/importer.go index 749c895..7c85fdd 100644 --- a/importer.go +++ b/importer.go @@ -132,7 +132,7 @@ func NewImportResult(rdres *RDWebResult) *ImportResult { return res } -func add_cart(ctx *ImportContext) (result *RDWebResult, err error) { +func add_cart(ctx *ImportContext) (result interface{}, err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -157,11 +157,20 @@ func add_cart(ctx *ImportContext) (result *RDWebResult, err error) { } } w.Close() - return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) - // TODO: on success this returns ... + + var res *http.Response + if res, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return NewRDWebResultFromXML(res.Body) + } + return NewRDCartAddFromXML(res.Body) } -func add_cut(ctx *ImportContext) (result *RDWebResult, err error) { +func add_cut(ctx *ImportContext) (result interface{}, err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -178,8 +187,17 @@ func add_cut(ctx *ImportContext) (result *RDWebResult, err error) { return } w.Close() - return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) - // TODO: on success this returns ... + + var res *http.Response + if res, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return NewRDWebResultFromXML(res.Body) + } + return NewRDCutAddFromXML(res.Body) } func remove_cart(ctx *ImportContext) (result *RDWebResult, err error) { @@ -199,10 +217,16 @@ func remove_cart(ctx *ImportContext) (result *RDWebResult, err error) { return } w.Close() - return send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()) + + var res *http.Response + if res, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer res.Body.Close() + return NewRDWebResultFromXML(res.Body) } -func send_post_request(url string, b *bytes.Buffer, contenttype string) (result *RDWebResult, err error) { +func send_post_request(url string, b *bytes.Buffer, contenttype string) (res *http.Response, err error) { var req *http.Request if req, err = http.NewRequest("POST", url, b); err != nil { return @@ -212,14 +236,10 @@ func send_post_request(url string, b *bytes.Buffer, contenttype string) (result } client := &http.Client{} - var res *http.Response if res, err = client.Do(req); err != nil { return } - defer res.Body.Close() - - // TODO: some commands might return something else... - return NewRDWebResultFromXML(res.Body) + return } func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *curl.Form, err error) { -- cgit v0.10.2 From a2305488a2e14c76665007782db423ef5a7d3e43 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 12 Dec 2015 04:18:59 +0100 Subject: imporved result reporting diff --git a/importer.go b/importer.go index 7c85fdd..505fffd 100644 --- a/importer.go +++ b/importer.go @@ -122,17 +122,15 @@ type ImportResult struct { Cut int } -func NewImportResult(rdres *RDWebResult) *ImportResult { - res := new(ImportResult) - res.ResponseCode = rdres.ResponseCode - res.ErrorString = rdres.ErrorString +func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { + self.ResponseCode = rdres.ResponseCode + self.ErrorString = rdres.ErrorString if rdres.AudioConvertError != 0 { - res.ErrorString += fmt.Sprint(", Audio Convert Error: %d", rdres.AudioConvertError) + self.ErrorString += fmt.Sprint(", Audio Convert Error: %d", rdres.AudioConvertError) } - return res } -func add_cart(ctx *ImportContext) (result interface{}, err error) { +func add_cart(ctx *ImportContext, result *ImportResult) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -165,12 +163,25 @@ func add_cart(ctx *ImportContext) (result interface{}, err error) { defer res.Body.Close() if res.StatusCode != http.StatusOK { - return NewRDWebResultFromXML(res.Body) + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(res.Body); err != nil { + return + } + result.fromRDWebResult(rdres) + return } - return NewRDCartAddFromXML(res.Body) + var cartadd *RDCartAdd + if cartadd, err = NewRDCartAddFromXML(res.Body); err != nil { + return + } + result.ResponseCode = res.StatusCode + result.ErrorString = "OK" + result.Cart = cartadd.Carts[0].Number + ctx.Cart = result.Cart + return } -func add_cut(ctx *ImportContext) (result interface{}, err error) { +func add_cut(ctx *ImportContext, result *ImportResult) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -195,12 +206,25 @@ func add_cut(ctx *ImportContext) (result interface{}, err error) { defer res.Body.Close() if res.StatusCode != http.StatusOK { - return NewRDWebResultFromXML(res.Body) + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(res.Body); err != nil { + return + } + result.fromRDWebResult(rdres) + return + } + var cutadd *RDCutAdd + if cutadd, err = NewRDCutAddFromXML(res.Body); err != nil { + return } - return NewRDCutAddFromXML(res.Body) + result.ResponseCode = res.StatusCode + result.ErrorString = "OK" + result.Cut = cutadd.Cuts[0].Number + ctx.Cut = cutadd.Cuts[0].Number + return } -func remove_cart(ctx *ImportContext) (result *RDWebResult, err error) { +func remove_cart(ctx *ImportContext, result *ImportResult) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -223,7 +247,13 @@ func remove_cart(ctx *ImportContext) (result *RDWebResult, err error) { return } defer res.Body.Close() - return NewRDWebResultFromXML(res.Body) + + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(res.Body); err != nil { + return + } + result.fromRDWebResult(rdres) + return } func send_post_request(url string, b *bytes.Buffer, contenttype string) (res *http.Response, err error) { @@ -279,7 +309,7 @@ func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *cur return } -func import_audio(ctx *ImportContext) (result *RDWebResult, err error) { +func import_audio(ctx *ImportContext, result *ImportResult) (err error) { easy := curl.EasyInit() if easy != nil { @@ -321,7 +351,12 @@ func import_audio(ctx *ImportContext) (result *RDWebResult, err error) { // if status_code, err = easy.Getinfo(curl.INFO_RESPONSE_CODE); err != nil { // return // } - return NewRDWebResultFromXML(bufio.NewReader(&resbody)) + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(bufio.NewReader(&resbody)); err != nil { + return + } + result.fromRDWebResult(rdres) + return } else { err = fmt.Errorf("Error initializing libcurl") } @@ -361,16 +396,15 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } } - var rdres *RDWebResult - if rdres, err = import_audio(ctx); err != nil { + res = &ImportResult{} + if err = import_audio(ctx, res); err != nil { return } if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(2, "importing", 1.0, ctx.ProgressCallBackData) } - res = NewImportResult(rdres) - rhdl.Printf("StatusCode: %d, ErrorString: '%s'\n", res.ResponseCode, res.ErrorString) + rhdl.Printf("ImportResult: %+v\n", res) if res.ResponseCode == http.StatusOK { rhl.Println("ImportFile succesfully imported", ctx.SourceFile) -- cgit v0.10.2 From 4c00b8293c1e3974d818af8022bc858a48733ec4 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 12 Dec 2015 05:02:12 +0100 Subject: importer now knows which information is needed for what operation uint vs int for some values diff --git a/importer.go b/importer.go index 505fffd..f365ea5 100644 --- a/importer.go +++ b/importer.go @@ -46,10 +46,11 @@ type ImportContext struct { Password string Trusted bool ShowId int + ClearShowCarts bool GroupName string - Cart int - Cut int - Channels int + Cart uint + Cut uint + Channels uint NormalizationLevel int AutotrimLevel int UseMetaData bool @@ -69,6 +70,7 @@ func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *Impo ctx.Password = "" ctx.Trusted = false ctx.ShowId = 0 + ctx.ClearShowCarts = false ctx.GroupName = group ctx.Cart = 0 ctx.Cut = 0 @@ -118,8 +120,8 @@ func (ctx *ImportContext) getShowInfo() (err error) { type ImportResult struct { ResponseCode int ErrorString string - Cart int - Cut int + Cart uint + Cut uint } func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { @@ -397,11 +399,40 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } res = &ImportResult{} - if err = import_audio(ctx, res); err != nil { - return - } - if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(2, "importing", 1.0, ctx.ProgressCallBackData) + if ctx.ShowId != 0 { + res.ResponseCode = 500 + res.ErrorString = "Importing to shows using the show-id is not yet implemented" + // TODO: fetch info from dropboxes (cartlist, groupname, import-params) + // - if (ClearShowCarts == true): foreach(cart in cartlist): remove_cart(cart) [200 || 404 -> OK] + // - ctx.cart = 0 + // - add_cart(ctx, res) [200 -> OK] + // - add_cut(ctx, res) [200 -> OK] + // - import_audio(ctx, res) [200 -> OK] + return + } else if ctx.GroupName != "" && ctx.Cart == 0 { + res.ResponseCode = 500 + res.ErrorString = "Importing to music pools using the group name is not yet implemented" + // TODO: fetch info from dropboxes (import-params) + // - add_cart(ctx, res) [200 -> OK] + // - add_cut(ctx, res) [200 -> OK] + // - import_audio(ctx, res) [200 -> OK] + } else if ctx.Cart != 0 && ctx.Cut == 0 { + res.ResponseCode = 500 + res.ErrorString = "Importing to a Cart which might not exist is not yet implemented" + // TODO: (everything except Cut and ShowId must be in context) + // - remove_cart(ctx, res) [200 || 404 -> OK] + // - add_cart(ctx, res) [200 -> OK] + // - add_cut(ctx, res) [200 -> OK] + // - import_audio(ctx, res) [200 -> OK] + } else if ctx.Cart != 0 && ctx.Cut != 0 { + if err = import_audio(ctx, res); err != nil { + return + } + res.Cart = ctx.Cart + res.Cut = ctx.Cut + } else { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = "The request doesn't contain enough Information to be processed" } rhdl.Printf("ImportResult: %+v\n", res) diff --git a/rdxport_responses.go b/rdxport_responses.go index db309d9..392f58f 100644 --- a/rdxport_responses.go +++ b/rdxport_responses.go @@ -51,7 +51,7 @@ type RDCartAdd struct { } type RDCart struct { - Number int `xml:"number"` + Number uint `xml:"number"` Type string `xml:"type"` GroupName string `xml:"groupName"` Title string `xml:"title"` @@ -64,15 +64,15 @@ type RDCart struct { Publisher string `xml:"publisher"` Composer string `xml:"composer"` UserDefined string `xml:"userDefined"` - UsageCode int `xml:"usageCode"` + UsageCode uint `xml:"usageCode"` ForcedLength string `xml:"forcedLength"` AverageLength string `xml:"averageLength"` LengthDeviation string `xml:"lengthDeviation"` AverageSegueLength string `xml:"averageSegueLenth"` AverageHookLength string `xml:"averageHookLength"` - CutQuantity int `xml:"cutQuantity"` - LastCutPlayer int `xml:"lastCutPlayed"` - Validity int `xml:"validity"` + CutQuantity uint `xml:"cutQuantity"` + LastCutPlayed uint `xml:"lastCutPlayed"` + Validity uint `xml:"validity"` EnforceLength bool `xml:"enforceLength"` Asynchronous bool `xml:"asyncronous"` Owner string `xml:"owner"` @@ -95,8 +95,8 @@ type RDCutAdd struct { type RDCut struct { Name string `xml:"cutName"` - CartNumber int `xml:"cartNumber"` - Number int `xml:"cutNumber"` + CartNumber uint `xml:"cartNumber"` + Number uint `xml:"cutNumber"` EverGreen bool `xml:"evergreen"` Description string `xml:"description"` OutCue string `xml:"outcue"` @@ -118,9 +118,9 @@ type RDCut struct { OriginName string `xml:"originName"` Weight int `xml:"weight"` LastPlayDateTime string `xml:"lastPlayDatetime"` - PlayCounter int `xml:"playCounter"` - LocalCounter int `xml:"localCounter"` - Validiy int `xml:"validity"` + PlayCounter uint `xml:"playCounter"` + LocalCounter uint `xml:"localCounter"` + Validiy uint `xml:"validity"` CondingFormat int `xml:"codingFormat"` SampleRate int `xml:"sampleRate"` BitRate int `xml:"bitRate"` -- cgit v0.10.2 From a19f18af10ff791fad73744b025844cc15eb16d3 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 12 Dec 2015 05:18:41 +0100 Subject: some thoughts diff --git a/fetcher.go b/fetcher.go index 05074b1..a86f7ca 100644 --- a/fetcher.go +++ b/fetcher.go @@ -128,6 +128,11 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { return } +// TODO: check path to import from -> don't touch problematic files like /etc/shadow... +// the daemon shouldn't be running as a user who can do any harm anyway +// still: let's make a special configurable directory the local:/// dir +// and only allow absolute paths here which will be based on the +// 'local' directory func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { @@ -141,6 +146,11 @@ func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { type FetchFunc func(*ImportContext, *url.URL) (err error) + +// TODO: implement fetchers for: +// archiv:// +// public:// +// home:// ????? var ( fetchers = map[string]FetchFunc{ "local": FetchFileLocal, @@ -170,6 +180,7 @@ func fetcher_init() { } } +// TODO: make sure a (partially) fetched file get's deleted on error func FetchFile(ctx *ImportContext) (err error) { var uri *url.URL -- cgit v0.10.2 From c1bff94567a3f1bbfe631795608a24fa44394e76 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 13 Dec 2015 01:36:41 +0100 Subject: GroupName should now be only supplied for music pools added context SanityCheck before fetching files diff --git a/fetcher.go b/fetcher.go index a86f7ca..7d406b2 100644 --- a/fetcher.go +++ b/fetcher.go @@ -146,7 +146,6 @@ func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { type FetchFunc func(*ImportContext, *url.URL) (err error) - // TODO: implement fetchers for: // archiv:// // public:// diff --git a/importer.go b/importer.go index f365ea5..0fc5ecf 100644 --- a/importer.go +++ b/importer.go @@ -62,7 +62,7 @@ type ImportContext struct { ProgressCallBackData interface{} } -func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *ImportContext { +func NewImportContext(conf *Config, rddb *RdDb, user string) *ImportContext { ctx := new(ImportContext) ctx.Config = conf ctx.RdDb = rddb @@ -71,11 +71,11 @@ func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *Impo ctx.Trusted = false ctx.ShowId = 0 ctx.ClearShowCarts = false - ctx.GroupName = group + ctx.GroupName = "" ctx.Cart = 0 ctx.Cut = 0 ctx.Channels = 2 - ctx.NormalizationLevel = -1200 + ctx.NormalizationLevel = -12 ctx.AutotrimLevel = 0 ctx.UseMetaData = false ctx.SourceFile = "" @@ -86,6 +86,29 @@ func NewImportContext(conf *Config, rddb *RdDb, user string, group string) *Impo return ctx } +func (ctx *ImportContext) SanityCheck() error { + if ctx.UserName == "" { + return fmt.Errorf("empty Username is not allowed") + } + if ctx.Password == "" && !ctx.Trusted { + return fmt.Errorf("empty Password on untrusted control interface is not allowed") + } + if ctx.ShowId != 0 { + return nil + } + if ctx.GroupName != "" { + // TODO: check if GroupName is a music pool -> Error if not + return nil + } + if ctx.Cart == 0 { + return fmt.Errorf("either ShowId, PoolName or CartNumber must be supplied") + } + if ctx.Channels != 1 && ctx.Channels != 2 { + return fmt.Errorf("channles must be 1 or 2") + } + return nil +} + func (ctx *ImportContext) getPassword(cached bool) (err error) { req := getPasswordRequest{} req.username = ctx.UserName @@ -409,7 +432,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { // - add_cut(ctx, res) [200 -> OK] // - import_audio(ctx, res) [200 -> OK] return - } else if ctx.GroupName != "" && ctx.Cart == 0 { + } else if ctx.GroupName != "" { res.ResponseCode = 500 res.ErrorString = "Importing to music pools using the group name is not yet implemented" // TODO: fetch info from dropboxes (import-params) @@ -419,7 +442,8 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } else if ctx.Cart != 0 && ctx.Cut == 0 { res.ResponseCode = 500 res.ErrorString = "Importing to a Cart which might not exist is not yet implemented" - // TODO: (everything except Cut and ShowId must be in context) + // TODO: (everything except Cut, GroupName and ShowId must be in context) + // - ctx.GetGroupOfCart(ctx.Cart) // - remove_cart(ctx, res) [200 || 404 -> OK] // - add_cart(ctx, res) [200 -> OK] // - add_cut(ctx, res) [200 -> OK] -- cgit v0.10.2 From 2a2d9d2a574da4e21414698e8b9ed1d273f237bc Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 13 Dec 2015 02:19:19 +0100 Subject: importer can now fetch group of cart diff --git a/importer.go b/importer.go index 0fc5ecf..95f61f7 100644 --- a/importer.go +++ b/importer.go @@ -125,6 +125,21 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { return } +func (ctx *ImportContext) getGroupOfCart() (err error) { + req := getGroupOfCartRequest{} + req.cart = ctx.Cart + req.response = make(chan getGroupOfCartResult) + ctx.RdDb.getGroupOfCartChan <- req + + res := <-req.response + if res.err != nil { + err = res.err + return + } + ctx.GroupName = res.group + return +} + func (ctx *ImportContext) getShowInfo() (err error) { req := getShowInfoRequest{} req.showid = ctx.ShowId @@ -434,7 +449,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { return } else if ctx.GroupName != "" { res.ResponseCode = 500 - res.ErrorString = "Importing to music pools using the group name is not yet implemented" + res.ErrorString = "Importing to music pools is not yet implemented" // TODO: fetch info from dropboxes (import-params) // - add_cart(ctx, res) [200 -> OK] // - add_cut(ctx, res) [200 -> OK] @@ -443,7 +458,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { res.ResponseCode = 500 res.ErrorString = "Importing to a Cart which might not exist is not yet implemented" // TODO: (everything except Cut, GroupName and ShowId must be in context) - // - ctx.GetGroupOfCart(ctx.Cart) + // - ctx.getGroupOfCart(ctx.Cart) // - remove_cart(ctx, res) [200 || 404 -> OK] // - add_cart(ctx, res) [200 -> OK] // - add_cut(ctx, res) [200 -> OK] @@ -456,7 +471,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { res.Cut = ctx.Cut } else { res.ResponseCode = http.StatusBadRequest - res.ErrorString = "The request doesn't contain enough Information to be processed" + res.ErrorString = "The request doesn't contain enough information to be processed" } rhdl.Printf("ImportResult: %+v\n", res) diff --git a/rddb.go b/rddb.go index dd1dfec..85a3177 100644 --- a/rddb.go +++ b/rddb.go @@ -41,6 +41,16 @@ type getPasswordRequest struct { response chan getPasswordResult } +type getGroupOfCartResult struct { + group string + err error +} + +type getGroupOfCartRequest struct { + cart uint + response chan getGroupOfCartResult +} + type getShowInfoResult struct { title string carts map[int]int @@ -53,14 +63,16 @@ type getShowInfoRequest struct { } type RdDb struct { - dbh *sql.DB - password_cache map[string]string - getPasswordChan chan getPasswordRequest - getPasswordStmt *sql.Stmt - getShowInfoChan chan getShowInfoRequest - getShowInfoStmt *sql.Stmt - quit chan bool - done chan bool + dbh *sql.DB + password_cache map[string]string + getPasswordChan chan getPasswordRequest + getPasswordStmt *sql.Stmt + getGroupOfCartChan chan getGroupOfCartRequest + getGroupOfCartStmt *sql.Stmt + getShowInfoChan chan getShowInfoRequest + getShowInfoStmt *sql.Stmt + quit chan bool + done chan bool } func (self *RdDb) init(conf *Config) (err error) { @@ -71,6 +83,9 @@ func (self *RdDb) init(conf *Config) (err error) { if self.getPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?;"); err != nil { return } + if self.getGroupOfCartStmt, err = self.dbh.Prepare("select NAME,DEFAULT_LOW_CART,DEFAULT_HIGH_CART from GROUPS where DEFAULT_LOW_CART <= ? and DEFAULT_HIGH_CART >= ?;"); err != nil { + return + } if self.getShowInfoStmt, err = self.dbh.Prepare("select TITLE,MACROS from CART where NUMBER = ?;"); err != nil { return } @@ -78,7 +93,6 @@ func (self *RdDb) init(conf *Config) (err error) { } func (self *RdDb) getPassword(username string, cached bool) (pwd string, err error) { - if cached { pwd = self.password_cache[username] } @@ -97,6 +111,36 @@ func (self *RdDb) getPassword(username string, cached bool) (pwd string, err err return } +func (self *RdDb) getGroupOfCart(cart uint) (group string, err error) { + var rows *sql.Rows + if rows, err = self.getGroupOfCartStmt.Query(cart, cart); err != nil { + return + } + defer rows.Close() + size_min := ^uint(0) + for rows.Next() { + var name string + var low_cart, high_cart uint + if err = rows.Scan(&name, &low_cart, &high_cart); err != nil { + return + } + if high_cart >= low_cart { + size := (high_cart - low_cart) + 1 + if size_min > size { + group = name + size_min = size + } + } + } + if err = rows.Err(); err != nil { + return + } + if group == "" { + err = fmt.Errorf("cart is outside of all group cart ranges") + } + return +} + func (self *RdDb) getShowInfo(showid int) (title string, carts map[int]int, err error) { var macros string err = self.getShowInfoStmt.QueryRow(showid).Scan(&title, ¯os) @@ -127,6 +171,10 @@ func (self *RdDb) dispatchRequests() { } pwd, err := self.getPassword(req.username, req.cached) req.response <- getPasswordResult{pwd, err} + case req := <-self.getGroupOfCartChan: + rhdl.Println("RdDb: got getGroupOfCart request for", req.cart) + group, err := self.getGroupOfCart(req.cart) + req.response <- getGroupOfCartResult{group, err} case req := <-self.getShowInfoChan: rhdl.Println("RdDb: got getShowInfo request for", req.showid) title, carts, err := self.getShowInfo(req.showid) @@ -147,6 +195,9 @@ func (self *RdDb) Cleanup() { if self.getPasswordStmt != nil { self.getPasswordStmt.Close() } + if self.getGroupOfCartStmt != nil { + self.getGroupOfCartStmt.Close() + } if self.getShowInfoStmt != nil { self.getPasswordStmt.Close() } @@ -159,6 +210,7 @@ func NewRdDb(conf *Config) (db *RdDb, err error) { db.done = make(chan bool) db.password_cache = make(map[string]string) db.getPasswordChan = make(chan getPasswordRequest) + db.getGroupOfCartChan = make(chan getGroupOfCartRequest) db.getShowInfoChan = make(chan getShowInfoRequest) if err = db.init(conf); err != nil { -- cgit v0.10.2 From 036e7a314729bb473998127f0ad20d7e1c3b7709 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 13 Dec 2015 05:02:43 +0100 Subject: music pool group check works now diff --git a/importer.go b/importer.go index 95f61f7..8206082 100644 --- a/importer.go +++ b/importer.go @@ -97,7 +97,13 @@ func (ctx *ImportContext) SanityCheck() error { return nil } if ctx.GroupName != "" { - // TODO: check if GroupName is a music pool -> Error if not + ismusic, err := ctx.checkMusicGroup() + if err != nil { + return err + } + if !ismusic { + return fmt.Errorf("supplied GroupName is not a music pool") + } return nil } if ctx.Cart == 0 { @@ -118,14 +124,13 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { res := <-req.response if res.err != nil { - err = res.err - return + return res.err } ctx.Password = res.password - return + return nil } -func (ctx *ImportContext) getGroupOfCart() (err error) { +func (ctx *ImportContext) getGroupOfCart() error { req := getGroupOfCartRequest{} req.cart = ctx.Cart req.response = make(chan getGroupOfCartResult) @@ -133,11 +138,10 @@ func (ctx *ImportContext) getGroupOfCart() (err error) { res := <-req.response if res.err != nil { - err = res.err - return + return res.err } ctx.GroupName = res.group - return + return nil } func (ctx *ImportContext) getShowInfo() (err error) { @@ -155,6 +159,19 @@ func (ctx *ImportContext) getShowInfo() (err error) { return } +func (ctx *ImportContext) checkMusicGroup() (bool, error) { + req := checkMusicGroupRequest{} + req.group = ctx.GroupName + req.response = make(chan checkMusicGroupResult) + ctx.RdDb.checkMusicGroupChan <- req + + res := <-req.response + if res.err != nil { + return false, res.err + } + return res.ismusic, nil +} + type ImportResult struct { ResponseCode int ErrorString string diff --git a/rddb.go b/rddb.go index 85a3177..52d6e18 100644 --- a/rddb.go +++ b/rddb.go @@ -62,17 +62,29 @@ type getShowInfoRequest struct { response chan getShowInfoResult } +type checkMusicGroupResult struct { + ismusic bool + err error +} + +type checkMusicGroupRequest struct { + group string + response chan checkMusicGroupResult +} + type RdDb struct { - dbh *sql.DB - password_cache map[string]string - getPasswordChan chan getPasswordRequest - getPasswordStmt *sql.Stmt - getGroupOfCartChan chan getGroupOfCartRequest - getGroupOfCartStmt *sql.Stmt - getShowInfoChan chan getShowInfoRequest - getShowInfoStmt *sql.Stmt - quit chan bool - done chan bool + dbh *sql.DB + password_cache map[string]string + getPasswordChan chan getPasswordRequest + getPasswordStmt *sql.Stmt + getGroupOfCartChan chan getGroupOfCartRequest + getGroupOfCartStmt *sql.Stmt + getShowInfoChan chan getShowInfoRequest + getShowInfoStmt *sql.Stmt + checkMusicGroupChan chan checkMusicGroupRequest + checkMusicGroupStmt *sql.Stmt + quit chan bool + done chan bool } func (self *RdDb) init(conf *Config) (err error) { @@ -89,6 +101,9 @@ func (self *RdDb) init(conf *Config) (err error) { if self.getShowInfoStmt, err = self.dbh.Prepare("select TITLE,MACROS from CART where NUMBER = ?;"); err != nil { return } + if self.checkMusicGroupStmt, err = self.dbh.Prepare("select count(*) from DROPBOXES where GROUP_NAME = ? and SET_USER_DEFINED like \"M;%\";"); err != nil { + return + } return } @@ -152,11 +167,25 @@ func (self *RdDb) getShowInfo(showid int) (title string, carts map[int]int, err } rhdl.Printf("RdDb: macros for showid '%d' are set to '%s'", showid, macros) - // TODO: also fetch cart list + // TODO: also fetch cart list and import params (norm_level, ...) carts = make(map[int]int) return } +func (self *RdDb) checkMusicGroup(group string) (ismusic bool, err error) { + var cnt int + err = self.checkMusicGroupStmt.QueryRow(group).Scan(&cnt) + if err != nil { + if err == sql.ErrNoRows { + err = nil + ismusic = false + } + return + } + ismusic = cnt > 0 + return +} + func (self *RdDb) dispatchRequests() { defer func() { self.done <- true }() for { @@ -179,6 +208,10 @@ func (self *RdDb) dispatchRequests() { rhdl.Println("RdDb: got getShowInfo request for", req.showid) title, carts, err := self.getShowInfo(req.showid) req.response <- getShowInfoResult{title, carts, err} + case req := <-self.checkMusicGroupChan: + rhdl.Println("RdDb: got checkMusicGroup request for", req.group) + ismusic, err := self.checkMusicGroup(req.group) + req.response <- checkMusicGroupResult{ismusic, err} } } } @@ -199,7 +232,10 @@ func (self *RdDb) Cleanup() { self.getGroupOfCartStmt.Close() } if self.getShowInfoStmt != nil { - self.getPasswordStmt.Close() + self.getShowInfoStmt.Close() + } + if self.checkMusicGroupStmt != nil { + self.checkMusicGroupStmt.Close() } } @@ -212,6 +248,7 @@ func NewRdDb(conf *Config) (db *RdDb, err error) { db.getPasswordChan = make(chan getPasswordRequest) db.getGroupOfCartChan = make(chan getGroupOfCartRequest) db.getShowInfoChan = make(chan getShowInfoRequest) + db.checkMusicGroupChan = make(chan checkMusicGroupRequest) if err = db.init(conf); err != nil { return -- cgit v0.10.2 From c870bfae31ac530488b5a97498b21825de6f7faa Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Dec 2015 00:16:59 +0100 Subject: fetching show info works now diff --git a/fetcher.go b/fetcher.go index 7d406b2..ae7c910 100644 --- a/fetcher.go +++ b/fetcher.go @@ -115,6 +115,7 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { + err = fmt.Errorf("fetcher('%s'): %s", ctx.SourceUri, err) return } diff --git a/importer.go b/importer.go index 8206082..9a254fc 100644 --- a/importer.go +++ b/importer.go @@ -155,7 +155,12 @@ func (ctx *ImportContext) getShowInfo() (err error) { err = res.err return } - rhdl.Printf("Title of show %d is '%s'", ctx.ShowId, res.title) + + rhdl.Printf("Show %d:\n", ctx.ShowId) + rhdl.Printf(" Title: '%s'\n", res.title) + rhdl.Printf(" Normalization Level: %d\n", res.norm_lvl) + rhdl.Printf(" Autotrim Level: %d\n", res.trim_lvl) + rhdl.Printf(" Carts: %v\n", res.carts) return } @@ -401,13 +406,10 @@ func import_audio(ctx *ImportContext, result *ImportResult) (err error) { easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { + err = fmt.Errorf("importer: %s", err) return } - // var status_code interface{} - // if status_code, err = easy.Getinfo(curl.INFO_RESPONSE_CODE); err != nil { - // return - // } var rdres *RDWebResult if rdres, err = NewRDWebResultFromXML(bufio.NewReader(&resbody)); err != nil { return @@ -459,7 +461,6 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { res.ErrorString = "Importing to shows using the show-id is not yet implemented" // TODO: fetch info from dropboxes (cartlist, groupname, import-params) // - if (ClearShowCarts == true): foreach(cart in cartlist): remove_cart(cart) [200 || 404 -> OK] - // - ctx.cart = 0 // - add_cart(ctx, res) [200 -> OK] // - add_cut(ctx, res) [200 -> OK] // - import_audio(ctx, res) [200 -> OK] diff --git a/rddb.go b/rddb.go index 52d6e18..e945483 100644 --- a/rddb.go +++ b/rddb.go @@ -28,6 +28,12 @@ import ( "database/sql" "fmt" _ "github.com/go-sql-driver/mysql" + "regexp" + "strings" +) + +var ( + showMacroRe = regexp.MustCompile(`^LL 1 ([^ ]+) 0\!$`) ) type getPasswordResult struct { @@ -52,9 +58,11 @@ type getGroupOfCartRequest struct { } type getShowInfoResult struct { - title string - carts map[int]int - err error + title string + carts []int + norm_lvl int + trim_lvl int + err error } type getShowInfoRequest struct { @@ -92,13 +100,15 @@ func (self *RdDb) init(conf *Config) (err error) { if self.dbh, err = sql.Open("mysql", dsn); err != nil { return } + //TODO: check if we have a compatible DB version + if self.getPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?;"); err != nil { return } if self.getGroupOfCartStmt, err = self.dbh.Prepare("select NAME,DEFAULT_LOW_CART,DEFAULT_HIGH_CART from GROUPS where DEFAULT_LOW_CART <= ? and DEFAULT_HIGH_CART >= ?;"); err != nil { return } - if self.getShowInfoStmt, err = self.dbh.Prepare("select TITLE,MACROS from CART where NUMBER = ?;"); err != nil { + if self.getShowInfoStmt, err = self.dbh.Prepare("select CART.TITLE,CART.MACROS,DROPBOXES.NORMALIZATION_LEVEL,DROPBOXES.AUTOTRIM_LEVEL,GROUPS.DEFAULT_LOW_CART,GROUPS.DEFAULT_HIGH_CART from CART, DROPBOXES, GROUPS where CART.NUMBER = DROPBOXES.TO_CART and GROUPS.NAME = DROPBOXES.GROUP_NAME and CART.NUMBER = ?;"); err != nil { return } if self.checkMusicGroupStmt, err = self.dbh.Prepare("select count(*) from DROPBOXES where GROUP_NAME = ? and SET_USER_DEFINED like \"M;%\";"); err != nil { @@ -156,19 +166,41 @@ func (self *RdDb) getGroupOfCart(cart uint) (group string, err error) { return } -func (self *RdDb) getShowInfo(showid int) (title string, carts map[int]int, err error) { +func (self *RdDb) getLogTableName(log string) string { + return strings.Replace(log, " ", "_", -1) + "_LOG" // TODO: this should get escaped fir mySQL but golang doesn't support it!!! +} + +func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []int, err error) { + q := fmt.Sprintf("select CART_NUMBER from %s where CART_NUMBER >= %d and CART_NUMBER <= %d order by COUNT;", self.getLogTableName(log), low_cart, high_cart) + var rows *sql.Rows + if rows, err = self.dbh.Query(q); err != nil { + return + } + defer rows.Close() + for rows.Next() { + var cart int + if err = rows.Scan(&cart); err != nil { + return + } + carts = append(carts, cart) + } + err = rows.Err() + return +} + +func (self *RdDb) getShowInfo(showid int) (title string, carts []int, norm_lvl int, trim_lvl int, err error) { var macros string - err = self.getShowInfoStmt.QueryRow(showid).Scan(&title, ¯os) + var low_cart, high_cart int + err = self.getShowInfoStmt.QueryRow(showid).Scan(&title, ¯os, &norm_lvl, &trim_lvl, &low_cart, &high_cart) if err != nil { if err == sql.ErrNoRows { err = fmt.Errorf("show '%d' not found", showid) } return } - - rhdl.Printf("RdDb: macros for showid '%d' are set to '%s'", showid, macros) - // TODO: also fetch cart list and import params (norm_level, ...) - carts = make(map[int]int) + norm_lvl /= 100 + trim_lvl /= 100 + carts, err = self.getShowCarts(showMacroRe.FindStringSubmatch(macros)[1], low_cart, high_cart) return } @@ -193,23 +225,15 @@ func (self *RdDb) dispatchRequests() { case <-self.quit: return case req := <-self.getPasswordChan: - if req.cached { - rhdl.Println("RdDb: got getPassword request for", req.username, "(cached)") - } else { - rhdl.Println("RdDb: got getPassword request for", req.username, "(not cached)") - } pwd, err := self.getPassword(req.username, req.cached) req.response <- getPasswordResult{pwd, err} case req := <-self.getGroupOfCartChan: - rhdl.Println("RdDb: got getGroupOfCart request for", req.cart) group, err := self.getGroupOfCart(req.cart) req.response <- getGroupOfCartResult{group, err} case req := <-self.getShowInfoChan: - rhdl.Println("RdDb: got getShowInfo request for", req.showid) - title, carts, err := self.getShowInfo(req.showid) - req.response <- getShowInfoResult{title, carts, err} + title, carts, norm_lvl, trim_lvl, err := self.getShowInfo(req.showid) + req.response <- getShowInfoResult{title, carts, norm_lvl, trim_lvl, err} case req := <-self.checkMusicGroupChan: - rhdl.Println("RdDb: got checkMusicGroup request for", req.group) ismusic, err := self.checkMusicGroup(req.group) req.response <- checkMusicGroupResult{ismusic, err} } -- cgit v0.10.2 From 7c58b88bd5cdfc667b8e5d30733e4eb7ce1a02c6 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Dec 2015 00:54:44 +0100 Subject: implemented DB version check diff --git a/rddb.go b/rddb.go index e945483..03553b9 100644 --- a/rddb.go +++ b/rddb.go @@ -36,6 +36,10 @@ var ( showMacroRe = regexp.MustCompile(`^LL 1 ([^ ]+) 0\!$`) ) +const ( + DB_VERSION = 245 +) + type getPasswordResult struct { password string err error @@ -96,11 +100,21 @@ type RdDb struct { } func (self *RdDb) init(conf *Config) (err error) { - dsn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s", conf.db_user, conf.db_passwd, conf.db_host, conf.db_db) + dsn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s?charset=utf8", conf.db_user, conf.db_passwd, conf.db_host, conf.db_db) if self.dbh, err = sql.Open("mysql", dsn); err != nil { return } - //TODO: check if we have a compatible DB version + + var dbver int + err = self.dbh.QueryRow("select DB from VERSION;").Scan(&dbver) + if err != nil { + err = fmt.Errorf("fetching version: %s", err) + return + } + if dbver != DB_VERSION { + err = fmt.Errorf("version mismatch is %d, should be %d", dbver, DB_VERSION) + return + } if self.getPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?;"); err != nil { return @@ -167,7 +181,7 @@ func (self *RdDb) getGroupOfCart(cart uint) (group string, err error) { } func (self *RdDb) getLogTableName(log string) string { - return strings.Replace(log, " ", "_", -1) + "_LOG" // TODO: this should get escaped fir mySQL but golang doesn't support it!!! + return strings.Replace(log, " ", "_", -1) + "_LOG" // TODO: this should get escaped for mySQL but golang doesn't support it!!! } func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []int, err error) { -- cgit v0.10.2 From ef771e903bb7c799de96224591dc7d46847bd47a Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Dec 2015 02:57:21 +0100 Subject: fetch Music Pool import-params diff --git a/importer.go b/importer.go index 9a254fc..61c2b64 100644 --- a/importer.go +++ b/importer.go @@ -177,6 +177,21 @@ func (ctx *ImportContext) checkMusicGroup() (bool, error) { return res.ismusic, nil } +func (ctx *ImportContext) getMusicInfo() (err error) { + req := getMusicInfoRequest{} + req.group = ctx.GroupName + req.response = make(chan getMusicInfoResult) + ctx.RdDb.getMusicInfoChan <- req + + res := <-req.response + if res.err != nil { + return res.err + } + ctx.NormalizationLevel = res.norm_lvl + ctx.AutotrimLevel = res.trim_lvl + return +} + type ImportResult struct { ResponseCode int ErrorString string @@ -460,10 +475,10 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { res.ResponseCode = 500 res.ErrorString = "Importing to shows using the show-id is not yet implemented" // TODO: fetch info from dropboxes (cartlist, groupname, import-params) + // - if (ctx.Cart not in cartlist) -> Error // - if (ClearShowCarts == true): foreach(cart in cartlist): remove_cart(cart) [200 || 404 -> OK] // - add_cart(ctx, res) [200 -> OK] // - add_cut(ctx, res) [200 -> OK] - // - import_audio(ctx, res) [200 -> OK] return } else if ctx.GroupName != "" { res.ResponseCode = 500 @@ -471,7 +486,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { // TODO: fetch info from dropboxes (import-params) // - add_cart(ctx, res) [200 -> OK] // - add_cut(ctx, res) [200 -> OK] - // - import_audio(ctx, res) [200 -> OK] + return } else if ctx.Cart != 0 && ctx.Cut == 0 { res.ResponseCode = 500 res.ErrorString = "Importing to a Cart which might not exist is not yet implemented" @@ -480,8 +495,10 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { // - remove_cart(ctx, res) [200 || 404 -> OK] // - add_cart(ctx, res) [200 -> OK] // - add_cut(ctx, res) [200 -> OK] - // - import_audio(ctx, res) [200 -> OK] - } else if ctx.Cart != 0 && ctx.Cut != 0 { + return + } + + if ctx.Cart != 0 && ctx.Cut != 0 { if err = import_audio(ctx, res); err != nil { return } diff --git a/rddb.go b/rddb.go index 03553b9..87ac17a 100644 --- a/rddb.go +++ b/rddb.go @@ -84,6 +84,17 @@ type checkMusicGroupRequest struct { response chan checkMusicGroupResult } +type getMusicInfoResult struct { + norm_lvl int + trim_lvl int + err error +} + +type getMusicInfoRequest struct { + group string + response chan getMusicInfoResult +} + type RdDb struct { dbh *sql.DB password_cache map[string]string @@ -95,6 +106,8 @@ type RdDb struct { getShowInfoStmt *sql.Stmt checkMusicGroupChan chan checkMusicGroupRequest checkMusicGroupStmt *sql.Stmt + getMusicInfoChan chan getMusicInfoRequest + getMusicInfoStmt *sql.Stmt quit chan bool done chan bool } @@ -128,6 +141,9 @@ func (self *RdDb) init(conf *Config) (err error) { if self.checkMusicGroupStmt, err = self.dbh.Prepare("select count(*) from DROPBOXES where GROUP_NAME = ? and SET_USER_DEFINED like \"M;%\";"); err != nil { return } + if self.getMusicInfoStmt, err = self.dbh.Prepare("select NORMALIZATION_LEVEL,AUTOTRIM_LEVEL from DROPBOXES where DROPBOXES.GROUP_NAME = ?;"); err != nil { + return + } return } @@ -220,7 +236,7 @@ func (self *RdDb) getShowInfo(showid int) (title string, carts []int, norm_lvl i func (self *RdDb) checkMusicGroup(group string) (ismusic bool, err error) { var cnt int - err = self.checkMusicGroupStmt.QueryRow(group).Scan(&cnt) + err = self.getMusicInfoStmt.QueryRow(group).Scan(&cnt) if err != nil { if err == sql.ErrNoRows { err = nil @@ -232,6 +248,17 @@ func (self *RdDb) checkMusicGroup(group string) (ismusic bool, err error) { return } +func (self *RdDb) getMusicInfo(group string) (norm_lvl, trim_lvl int, err error) { + err = self.checkMusicGroupStmt.QueryRow(group).Scan(&norm_lvl, &trim_lvl) + if err != nil { + if err == sql.ErrNoRows { + err = fmt.Errorf("music pool '%d' not found", group) + } + return + } + return +} + func (self *RdDb) dispatchRequests() { defer func() { self.done <- true }() for { @@ -250,6 +277,9 @@ func (self *RdDb) dispatchRequests() { case req := <-self.checkMusicGroupChan: ismusic, err := self.checkMusicGroup(req.group) req.response <- checkMusicGroupResult{ismusic, err} + case req := <-self.getMusicInfoChan: + norm_lvl, trim_lvl, err := self.getMusicInfo(req.group) + req.response <- getMusicInfoResult{norm_lvl, trim_lvl, err} } } } @@ -275,6 +305,9 @@ func (self *RdDb) Cleanup() { if self.checkMusicGroupStmt != nil { self.checkMusicGroupStmt.Close() } + if self.getMusicInfoStmt != nil { + self.getMusicInfoStmt.Close() + } } func NewRdDb(conf *Config) (db *RdDb, err error) { @@ -287,6 +320,7 @@ func NewRdDb(conf *Config) (db *RdDb, err error) { db.getGroupOfCartChan = make(chan getGroupOfCartRequest) db.getShowInfoChan = make(chan getShowInfoRequest) db.checkMusicGroupChan = make(chan checkMusicGroupRequest) + db.getMusicInfoChan = make(chan getMusicInfoRequest) if err = db.init(conf); err != nil { return -- cgit v0.10.2 From cd522b26ded18ef1972647ac8353f6549468e903 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Dec 2015 03:20:59 +0100 Subject: minor refactoring at RdDb diff --git a/rddb.go b/rddb.go index 87ac17a..0a9e21e 100644 --- a/rddb.go +++ b/rddb.go @@ -147,28 +147,27 @@ func (self *RdDb) init(conf *Config) (err error) { return } -func (self *RdDb) getPassword(username string, cached bool) (pwd string, err error) { +func (self *RdDb) getPassword(username string, cached bool) (result getPasswordResult) { if cached { - pwd = self.password_cache[username] + result.password = self.password_cache[username] } - if pwd == "" { - err = self.getPasswordStmt.QueryRow(username).Scan(&pwd) - if err != nil { - if err == sql.ErrNoRows { - err = fmt.Errorf("user '%s' not known by rivendell", username) + if result.password == "" { + if result.err = self.getPasswordStmt.QueryRow(username).Scan(&result.password); result.err != nil { + if result.err == sql.ErrNoRows { + result.err = fmt.Errorf("user '%s' not known by rivendell", username) } return } - self.password_cache[username] = pwd + self.password_cache[username] = result.password } return } -func (self *RdDb) getGroupOfCart(cart uint) (group string, err error) { +func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { var rows *sql.Rows - if rows, err = self.getGroupOfCartStmt.Query(cart, cart); err != nil { + if rows, result.err = self.getGroupOfCartStmt.Query(cart, cart); result.err != nil { return } defer rows.Close() @@ -176,22 +175,22 @@ func (self *RdDb) getGroupOfCart(cart uint) (group string, err error) { for rows.Next() { var name string var low_cart, high_cart uint - if err = rows.Scan(&name, &low_cart, &high_cart); err != nil { + if result.err = rows.Scan(&name, &low_cart, &high_cart); result.err != nil { return } if high_cart >= low_cart { size := (high_cart - low_cart) + 1 if size_min > size { - group = name + result.group = name size_min = size } } } - if err = rows.Err(); err != nil { + if result.err = rows.Err(); result.err != nil { return } - if group == "" { - err = fmt.Errorf("cart is outside of all group cart ranges") + if result.group == "" { + result.err = fmt.Errorf("cart is outside of all group cart ranges") } return } @@ -218,41 +217,40 @@ func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []int return } -func (self *RdDb) getShowInfo(showid int) (title string, carts []int, norm_lvl int, trim_lvl int, err error) { +func (self *RdDb) getShowInfo(showid int) (result getShowInfoResult) { var macros string var low_cart, high_cart int - err = self.getShowInfoStmt.QueryRow(showid).Scan(&title, ¯os, &norm_lvl, &trim_lvl, &low_cart, &high_cart) - if err != nil { - if err == sql.ErrNoRows { - err = fmt.Errorf("show '%d' not found", showid) + result.err = self.getShowInfoStmt.QueryRow(showid).Scan(&result.title, ¯os, &result.norm_lvl, &result.trim_lvl, &low_cart, &high_cart) + if result.err != nil { + if result.err == sql.ErrNoRows { + result.err = fmt.Errorf("show '%d' not found", showid) } return } - norm_lvl /= 100 - trim_lvl /= 100 - carts, err = self.getShowCarts(showMacroRe.FindStringSubmatch(macros)[1], low_cart, high_cart) + result.norm_lvl /= 100 + result.trim_lvl /= 100 + result.carts, result.err = self.getShowCarts(showMacroRe.FindStringSubmatch(macros)[1], low_cart, high_cart) return } -func (self *RdDb) checkMusicGroup(group string) (ismusic bool, err error) { +func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { var cnt int - err = self.getMusicInfoStmt.QueryRow(group).Scan(&cnt) - if err != nil { - if err == sql.ErrNoRows { - err = nil - ismusic = false + if result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&cnt); result.err != nil { + if result.err == sql.ErrNoRows { + result.err = nil + result.ismusic = false } return } - ismusic = cnt > 0 + result.ismusic = cnt > 0 return } -func (self *RdDb) getMusicInfo(group string) (norm_lvl, trim_lvl int, err error) { - err = self.checkMusicGroupStmt.QueryRow(group).Scan(&norm_lvl, &trim_lvl) - if err != nil { - if err == sql.ErrNoRows { - err = fmt.Errorf("music pool '%d' not found", group) +func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { + result.err = self.checkMusicGroupStmt.QueryRow(group).Scan(&result.norm_lvl, &result.trim_lvl) + if result.err != nil { + if result.err == sql.ErrNoRows { + result.err = fmt.Errorf("music pool '%d' not found", group) } return } @@ -266,20 +264,15 @@ func (self *RdDb) dispatchRequests() { case <-self.quit: return case req := <-self.getPasswordChan: - pwd, err := self.getPassword(req.username, req.cached) - req.response <- getPasswordResult{pwd, err} + req.response <- self.getPassword(req.username, req.cached) case req := <-self.getGroupOfCartChan: - group, err := self.getGroupOfCart(req.cart) - req.response <- getGroupOfCartResult{group, err} + req.response <- self.getGroupOfCart(req.cart) case req := <-self.getShowInfoChan: - title, carts, norm_lvl, trim_lvl, err := self.getShowInfo(req.showid) - req.response <- getShowInfoResult{title, carts, norm_lvl, trim_lvl, err} + req.response <- self.getShowInfo(req.showid) case req := <-self.checkMusicGroupChan: - ismusic, err := self.checkMusicGroup(req.group) - req.response <- checkMusicGroupResult{ismusic, err} + req.response <- self.checkMusicGroup(req.group) case req := <-self.getMusicInfoChan: - norm_lvl, trim_lvl, err := self.getMusicInfo(req.group) - req.response <- getMusicInfoResult{norm_lvl, trim_lvl, err} + req.response <- self.getMusicInfo(req.group) } } } -- cgit v0.10.2 From 6e9e940372b1e76ecde2f6918c53592d5addfaf0 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Dec 2015 05:02:37 +0100 Subject: added more sophisticated import options - still needs testing diff --git a/importer.go b/importer.go index 61c2b64..925722e 100644 --- a/importer.go +++ b/importer.go @@ -45,7 +45,7 @@ type ImportContext struct { UserName string Password string Trusted bool - ShowId int + ShowId uint ClearShowCarts bool GroupName string Cart uint @@ -144,7 +144,7 @@ func (ctx *ImportContext) getGroupOfCart() error { return nil } -func (ctx *ImportContext) getShowInfo() (err error) { +func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { req := getShowInfoRequest{} req.showid = ctx.ShowId req.response = make(chan getShowInfoResult) @@ -155,12 +155,11 @@ func (ctx *ImportContext) getShowInfo() (err error) { err = res.err return } - - rhdl.Printf("Show %d:\n", ctx.ShowId) - rhdl.Printf(" Title: '%s'\n", res.title) - rhdl.Printf(" Normalization Level: %d\n", res.norm_lvl) - rhdl.Printf(" Autotrim Level: %d\n", res.trim_lvl) - rhdl.Printf(" Carts: %v\n", res.carts) + ctx.GroupName = res.group + ctx.NormalizationLevel = res.norm_lvl + ctx.AutotrimLevel = res.trim_lvl + ctx.UseMetaData = true + carts = res.carts return } @@ -189,6 +188,7 @@ func (ctx *ImportContext) getMusicInfo() (err error) { } ctx.NormalizationLevel = res.norm_lvl ctx.AutotrimLevel = res.trim_lvl + ctx.UseMetaData = true return } @@ -203,11 +203,11 @@ func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { self.ResponseCode = rdres.ResponseCode self.ErrorString = rdres.ErrorString if rdres.AudioConvertError != 0 { - self.ErrorString += fmt.Sprint(", Audio Convert Error: %d", rdres.AudioConvertError) + self.ErrorString += fmt.Sprintf(", Audio Convert Error: %d", rdres.AudioConvertError) } } -func add_cart(ctx *ImportContext, result *ImportResult) (err error) { +func add_cart(ctx *ImportContext, res *ImportResult) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -220,7 +220,7 @@ func add_cart(ctx *ImportContext, result *ImportResult) (err error) { if err = w.WriteField("PASSWORD", ctx.Password); err != nil { return } - if err = w.WriteField("GRPOUP_NAME", ctx.GroupName); err != nil { + if err = w.WriteField("GROUP_NAME", ctx.GroupName); err != nil { return } if err = w.WriteField("TYPE", "audio"); err != nil { @@ -233,32 +233,32 @@ func add_cart(ctx *ImportContext, result *ImportResult) (err error) { } w.Close() - var res *http.Response - if res, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + var resp *http.Response + if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } - defer res.Body.Close() + defer resp.Body.Close() - if res.StatusCode != http.StatusOK { + if resp.StatusCode != http.StatusOK { var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(res.Body); err != nil { + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { return } - result.fromRDWebResult(rdres) + res.fromRDWebResult(rdres) return } var cartadd *RDCartAdd - if cartadd, err = NewRDCartAddFromXML(res.Body); err != nil { + if cartadd, err = NewRDCartAddFromXML(resp.Body); err != nil { return } - result.ResponseCode = res.StatusCode - result.ErrorString = "OK" - result.Cart = cartadd.Carts[0].Number - ctx.Cart = result.Cart + res.ResponseCode = resp.StatusCode + res.ErrorString = "OK" + res.Cart = cartadd.Carts[0].Number + ctx.Cart = res.Cart return } -func add_cut(ctx *ImportContext, result *ImportResult) (err error) { +func add_cut(ctx *ImportContext, res *ImportResult) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -276,32 +276,32 @@ func add_cut(ctx *ImportContext, result *ImportResult) (err error) { } w.Close() - var res *http.Response - if res, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + var resp *http.Response + if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } - defer res.Body.Close() + defer resp.Body.Close() - if res.StatusCode != http.StatusOK { + if resp.StatusCode != http.StatusOK { var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(res.Body); err != nil { + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { return } - result.fromRDWebResult(rdres) + res.fromRDWebResult(rdres) return } var cutadd *RDCutAdd - if cutadd, err = NewRDCutAddFromXML(res.Body); err != nil { + if cutadd, err = NewRDCutAddFromXML(resp.Body); err != nil { return } - result.ResponseCode = res.StatusCode - result.ErrorString = "OK" - result.Cut = cutadd.Cuts[0].Number + res.ResponseCode = resp.StatusCode + res.ErrorString = "OK" + res.Cut = cutadd.Cuts[0].Number ctx.Cut = cutadd.Cuts[0].Number return } -func remove_cart(ctx *ImportContext, result *ImportResult) (err error) { +func remove_cart(ctx *ImportContext, res *ImportResult) (err error) { var b bytes.Buffer w := multipart.NewWriter(&b) @@ -319,17 +319,17 @@ func remove_cart(ctx *ImportContext, result *ImportResult) (err error) { } w.Close() - var res *http.Response - if res, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + var resp *http.Response + if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } - defer res.Body.Close() + defer resp.Body.Close() var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(res.Body); err != nil { + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { return } - result.fromRDWebResult(rdres) + res.fromRDWebResult(rdres) return } @@ -438,6 +438,61 @@ func import_audio(ctx *ImportContext, result *ImportResult) (err error) { return } +func remove_add_cart_cut(ctx *ImportContext, res *ImportResult) (err error) { + if err = remove_cart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + return + } + if err = add_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + return add_cut(ctx, res) +} + +func is_cart_member_of_show(ctx *ImportContext, res *ImportResult, carts []uint) (found bool) { + if ctx.Cart == 0 { + return true + } + for _, cart := range carts { + if cart == ctx.Cart { + return true + } + } + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("Requested cart %d is not a member of show: %d", ctx.Cart, ctx.ShowId) + return false +} + +func clear_show_carts(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { + if ctx.ClearShowCarts { + orig_cart := ctx.Cart + for _, cart := range carts { + ctx.Cart = cart + if err = remove_cart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + return + } + } + ctx.Cart = orig_cart + } + return +} + +func add_show_cart_cut(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { + if err = add_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + for _, cart := range carts { + if cart == ctx.Cart { + return add_cut(ctx, res) + } + } + if err = remove_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + res.ResponseCode = http.StatusForbidden + res.ErrorString = fmt.Sprintf("Show %d has no free carts left", ctx.ShowId) + return +} + func cleanup_files(ctx *ImportContext) { if ctx.DeleteSourceFile { rhdl.Printf("importer: removing file: %s", ctx.SourceFile) @@ -457,7 +512,6 @@ func cleanup_files(ctx *ImportContext) { } func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { - rhl.Println("ImportFile called for", ctx.SourceFile) defer cleanup_files(ctx) if ctx.ProgressCallBack != nil { @@ -470,32 +524,35 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } } - res = &ImportResult{} + res = &ImportResult{ResponseCode: http.StatusOK} if ctx.ShowId != 0 { - res.ResponseCode = 500 - res.ErrorString = "Importing to shows using the show-id is not yet implemented" - // TODO: fetch info from dropboxes (cartlist, groupname, import-params) - // - if (ctx.Cart not in cartlist) -> Error - // - if (ClearShowCarts == true): foreach(cart in cartlist): remove_cart(cart) [200 || 404 -> OK] - // - add_cart(ctx, res) [200 -> OK] - // - add_cut(ctx, res) [200 -> OK] - return + var show_carts []uint + if show_carts, err = ctx.getShowInfo(); err != nil { + return + } + if !is_cart_member_of_show(ctx, res, show_carts) { + return + } + if err = clear_show_carts(ctx, res, show_carts); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + return + } + if err = add_show_cart_cut(ctx, res, show_carts); err != nil || res.ResponseCode != http.StatusOK { + return + } } else if ctx.GroupName != "" { - res.ResponseCode = 500 - res.ErrorString = "Importing to music pools is not yet implemented" - // TODO: fetch info from dropboxes (import-params) - // - add_cart(ctx, res) [200 -> OK] - // - add_cut(ctx, res) [200 -> OK] - return - } else if ctx.Cart != 0 && ctx.Cut == 0 { - res.ResponseCode = 500 - res.ErrorString = "Importing to a Cart which might not exist is not yet implemented" - // TODO: (everything except Cut, GroupName and ShowId must be in context) - // - ctx.getGroupOfCart(ctx.Cart) - // - remove_cart(ctx, res) [200 || 404 -> OK] - // - add_cart(ctx, res) [200 -> OK] - // - add_cut(ctx, res) [200 -> OK] - return + if err = ctx.getMusicInfo(); err != nil { + return + } + if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + } else if ctx.Cart != 0 && ctx.Cut == 0 { // TODO: we should add a DeleteCart option... + if err = ctx.getGroupOfCart(); err != nil { + return + } + if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } } if ctx.Cart != 0 && ctx.Cut != 0 { @@ -510,11 +567,5 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } rhdl.Printf("ImportResult: %+v\n", res) - - if res.ResponseCode == http.StatusOK { - rhl.Println("ImportFile succesfully imported", ctx.SourceFile) - } else { - rhl.Println("ImportFile import of", ctx.SourceFile, "was unsuccesful") - } return } diff --git a/rddb.go b/rddb.go index 0a9e21e..b693c47 100644 --- a/rddb.go +++ b/rddb.go @@ -63,14 +63,15 @@ type getGroupOfCartRequest struct { type getShowInfoResult struct { title string - carts []int + group string + carts []uint norm_lvl int trim_lvl int err error } type getShowInfoRequest struct { - showid int + showid uint response chan getShowInfoResult } @@ -135,7 +136,7 @@ func (self *RdDb) init(conf *Config) (err error) { if self.getGroupOfCartStmt, err = self.dbh.Prepare("select NAME,DEFAULT_LOW_CART,DEFAULT_HIGH_CART from GROUPS where DEFAULT_LOW_CART <= ? and DEFAULT_HIGH_CART >= ?;"); err != nil { return } - if self.getShowInfoStmt, err = self.dbh.Prepare("select CART.TITLE,CART.MACROS,DROPBOXES.NORMALIZATION_LEVEL,DROPBOXES.AUTOTRIM_LEVEL,GROUPS.DEFAULT_LOW_CART,GROUPS.DEFAULT_HIGH_CART from CART, DROPBOXES, GROUPS where CART.NUMBER = DROPBOXES.TO_CART and GROUPS.NAME = DROPBOXES.GROUP_NAME and CART.NUMBER = ?;"); err != nil { + if self.getShowInfoStmt, err = self.dbh.Prepare("select CART.TITLE,CART.MACROS,DROPBOXES.GROUP_NAME,DROPBOXES.NORMALIZATION_LEVEL,DROPBOXES.AUTOTRIM_LEVEL,GROUPS.DEFAULT_LOW_CART,GROUPS.DEFAULT_HIGH_CART from CART, DROPBOXES, GROUPS where CART.NUMBER = DROPBOXES.TO_CART and GROUPS.NAME = DROPBOXES.GROUP_NAME and CART.NUMBER = ?;"); err != nil { return } if self.checkMusicGroupStmt, err = self.dbh.Prepare("select count(*) from DROPBOXES where GROUP_NAME = ? and SET_USER_DEFINED like \"M;%\";"); err != nil { @@ -199,7 +200,7 @@ func (self *RdDb) getLogTableName(log string) string { return strings.Replace(log, " ", "_", -1) + "_LOG" // TODO: this should get escaped for mySQL but golang doesn't support it!!! } -func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []int, err error) { +func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []uint, err error) { q := fmt.Sprintf("select CART_NUMBER from %s where CART_NUMBER >= %d and CART_NUMBER <= %d order by COUNT;", self.getLogTableName(log), low_cart, high_cart) var rows *sql.Rows if rows, err = self.dbh.Query(q); err != nil { @@ -207,7 +208,7 @@ func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []int } defer rows.Close() for rows.Next() { - var cart int + var cart uint if err = rows.Scan(&cart); err != nil { return } @@ -217,10 +218,10 @@ func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []int return } -func (self *RdDb) getShowInfo(showid int) (result getShowInfoResult) { +func (self *RdDb) getShowInfo(showid uint) (result getShowInfoResult) { var macros string var low_cart, high_cart int - result.err = self.getShowInfoStmt.QueryRow(showid).Scan(&result.title, ¯os, &result.norm_lvl, &result.trim_lvl, &low_cart, &high_cart) + result.err = self.getShowInfoStmt.QueryRow(showid).Scan(&result.title, ¯os, &result.group, &result.norm_lvl, &result.trim_lvl, &low_cart, &high_cart) if result.err != nil { if result.err == sql.ErrNoRows { result.err = fmt.Errorf("show '%d' not found", showid) -- cgit v0.10.2 From 090f1621563a2b5069dd54ba7e67b7c07cc79f13 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Dec 2015 15:48:37 +0100 Subject: fixed bug where some context values where not initialiized properly diff --git a/conf.go b/conf.go index 7288a89..2386ba9 100644 --- a/conf.go +++ b/conf.go @@ -28,6 +28,13 @@ import ( "github.com/vaughan0/go-ini" ) +type ImportParamDefaults struct { + Channels uint + NormalizationLevel int + AutotrimLevel int + UseMetaData bool +} + type Config struct { configfile string RDXportEndpoint string @@ -36,6 +43,7 @@ type Config struct { db_user string db_passwd string db_db string + ImportParamDefaults } func get_ini_value(file ini.File, section string, key string, dflt string) string { @@ -67,5 +75,9 @@ func NewConfig(configfile, rdxport_endpoint, temp_dir *string) (conf *Config, er } conf.RDXportEndpoint = *rdxport_endpoint conf.TempDir = *temp_dir + conf.ImportParamDefaults.Channels = 2 + conf.ImportParamDefaults.NormalizationLevel = -12 + conf.ImportParamDefaults.AutotrimLevel = 0 + conf.ImportParamDefaults.UseMetaData = true return } diff --git a/importer.go b/importer.go index 925722e..aa5343d 100644 --- a/importer.go +++ b/importer.go @@ -74,10 +74,10 @@ func NewImportContext(conf *Config, rddb *RdDb, user string) *ImportContext { ctx.GroupName = "" ctx.Cart = 0 ctx.Cut = 0 - ctx.Channels = 2 - ctx.NormalizationLevel = -12 - ctx.AutotrimLevel = 0 - ctx.UseMetaData = false + ctx.Channels = conf.ImportParamDefaults.Channels + ctx.NormalizationLevel = conf.ImportParamDefaults.NormalizationLevel + ctx.AutotrimLevel = conf.ImportParamDefaults.AutotrimLevel + ctx.UseMetaData = conf.ImportParamDefaults.UseMetaData ctx.SourceFile = "" ctx.DeleteSourceFile = false ctx.DeleteSourceDir = false @@ -158,6 +158,7 @@ func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { ctx.GroupName = res.group ctx.NormalizationLevel = res.norm_lvl ctx.AutotrimLevel = res.trim_lvl + ctx.Channels = 2 ctx.UseMetaData = true carts = res.carts return @@ -188,6 +189,7 @@ func (ctx *ImportContext) getMusicInfo() (err error) { } ctx.NormalizationLevel = res.norm_lvl ctx.AutotrimLevel = res.trim_lvl + ctx.Channels = 2 ctx.UseMetaData = true return } diff --git a/rddb.go b/rddb.go index b693c47..8b72f58 100644 --- a/rddb.go +++ b/rddb.go @@ -251,7 +251,7 @@ func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { result.err = self.checkMusicGroupStmt.QueryRow(group).Scan(&result.norm_lvl, &result.trim_lvl) if result.err != nil { if result.err == sql.ErrNoRows { - result.err = fmt.Errorf("music pool '%d' not found", group) + result.err = fmt.Errorf("music pool '%s' not found", group) } return } -- cgit v0.10.2 From 638d17b5051f80b36ffa641366440b8266eaac4c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 14 Dec 2015 16:29:05 +0100 Subject: switched to mymysql go driver and added escape for log table name diff --git a/rddb.go b/rddb.go index 8b72f58..34019e8 100644 --- a/rddb.go +++ b/rddb.go @@ -27,13 +27,14 @@ package rhimport import ( "database/sql" "fmt" - _ "github.com/go-sql-driver/mysql" + "github.com/ziutek/mymysql/godrv" "regexp" "strings" ) var ( showMacroRe = regexp.MustCompile(`^LL 1 ([^ ]+) 0\!$`) + mysqlTableNameRe = regexp.MustCompile(`^[_0-9a-zA-Z-]+$`) ) const ( @@ -114,8 +115,10 @@ type RdDb struct { } func (self *RdDb) init(conf *Config) (err error) { - dsn := fmt.Sprintf("%s:%s@tcp(%s:3306)/%s?charset=utf8", conf.db_user, conf.db_passwd, conf.db_host, conf.db_db) - if self.dbh, err = sql.Open("mysql", dsn); err != nil { + godrv.Register("SET CHARACTER SET utf8;") + + dsn := fmt.Sprintf("tcp:%s:3306*%s/%s/%s", conf.db_host, conf.db_db, conf.db_user, conf.db_passwd) + if self.dbh, err = sql.Open("mymysql", dsn); err != nil { return } @@ -196,12 +199,20 @@ func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { return } -func (self *RdDb) getLogTableName(log string) string { - return strings.Replace(log, " ", "_", -1) + "_LOG" // TODO: this should get escaped for mySQL but golang doesn't support it!!! +func (self *RdDb) getLogTableName(log string) (logtable string, err error) { + logtable = strings.Replace(log, " ", "_", -1) + "_LOG" + if !mysqlTableNameRe.MatchString(logtable) { + return "", fmt.Errorf("the log table name contains illegal charecters: %s", logtable) + } + return } func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []uint, err error) { - q := fmt.Sprintf("select CART_NUMBER from %s where CART_NUMBER >= %d and CART_NUMBER <= %d order by COUNT;", self.getLogTableName(log), low_cart, high_cart) + var logtable string + if logtable, err = self.getLogTableName(log); err != nil { + return + } + q := fmt.Sprintf("select CART_NUMBER from %s where CART_NUMBER >= %d and CART_NUMBER <= %d order by COUNT;", logtable, low_cart, high_cart) var rows *sql.Rows if rows, err = self.dbh.Query(q); err != nil { return -- cgit v0.10.2 From c3d7a6f2ea59e75a5e2c3939eea6a5248d9f779a Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 15 Dec 2015 19:47:22 +0100 Subject: improved error handling and overall resilience to different situations diff --git a/fetcher.go b/fetcher.go index ae7c910..38e1d7f 100644 --- a/fetcher.go +++ b/fetcher.go @@ -134,6 +134,8 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { // still: let's make a special configurable directory the local:/// dir // and only allow absolute paths here which will be based on the // 'local' directory +// TODO: also check if file exists and is accessable!!! otherwise curl will blow up +// with a not-easy-to-understand error func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { diff --git a/importer.go b/importer.go index aa5343d..94065f6 100644 --- a/importer.go +++ b/importer.go @@ -49,6 +49,7 @@ type ImportContext struct { ClearShowCarts bool GroupName string Cart uint + ClearCart bool Cut uint Channels uint NormalizationLevel int @@ -73,6 +74,7 @@ func NewImportContext(conf *Config, rddb *RdDb, user string) *ImportContext { ctx.ClearShowCarts = false ctx.GroupName = "" ctx.Cart = 0 + ctx.ClearCart = false ctx.Cut = 0 ctx.Channels = conf.ImportParamDefaults.Channels ctx.NormalizationLevel = conf.ImportParamDefaults.NormalizationLevel @@ -210,6 +212,14 @@ func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { } func add_cart(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: add_cart() called for cart: %d", ctx.Cart) + + if ctx.GroupName == "" { + if err = ctx.getGroupOfCart(); err != nil { + return + } + } + var b bytes.Buffer w := multipart.NewWriter(&b) @@ -247,6 +257,7 @@ func add_cart(ctx *ImportContext, res *ImportResult) (err error) { return } res.fromRDWebResult(rdres) + res.Cart = ctx.Cart return } var cartadd *RDCartAdd @@ -261,6 +272,7 @@ func add_cart(ctx *ImportContext, res *ImportResult) (err error) { } func add_cut(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: add_cut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -290,6 +302,8 @@ func add_cut(ctx *ImportContext, res *ImportResult) (err error) { return } res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + res.Cut = ctx.Cut return } var cutadd *RDCutAdd @@ -298,12 +312,14 @@ func add_cut(ctx *ImportContext, res *ImportResult) (err error) { } res.ResponseCode = resp.StatusCode res.ErrorString = "OK" + res.Cart = ctx.Cart res.Cut = cutadd.Cuts[0].Number ctx.Cut = cutadd.Cuts[0].Number return } func remove_cart(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: remove_cart() called for cart: %d", ctx.Cart) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -332,10 +348,49 @@ func remove_cart(ctx *ImportContext, res *ImportResult) (err error) { return } res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + return +} + +func remove_cut(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: remove_cut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "11"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + if err = w.WriteField("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { + return + } + w.Close() + + var resp *http.Response + if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer resp.Body.Close() + + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { + return + } + res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + res.Cut = ctx.Cut return } -func send_post_request(url string, b *bytes.Buffer, contenttype string) (res *http.Response, err error) { +func send_post_request(url string, b *bytes.Buffer, contenttype string) (resp *http.Response, err error) { var req *http.Request if req, err = http.NewRequest("POST", url, b); err != nil { return @@ -345,7 +400,7 @@ func send_post_request(url string, b *bytes.Buffer, contenttype string) (res *ht } client := &http.Client{} - if res, err = client.Do(req); err != nil { + if resp, err = client.Do(req); err != nil { return } return @@ -388,7 +443,8 @@ func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *cur return } -func import_audio(ctx *ImportContext, result *ImportResult) (err error) { +func import_audio(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: import_audio() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) easy := curl.EasyInit() if easy != nil { @@ -431,8 +487,9 @@ func import_audio(ctx *ImportContext, result *ImportResult) (err error) { if rdres, err = NewRDWebResultFromXML(bufio.NewReader(&resbody)); err != nil { return } - result.fromRDWebResult(rdres) - return + res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + res.Cut = ctx.Cut } else { err = fmt.Errorf("Error initializing libcurl") } @@ -440,14 +497,21 @@ func import_audio(ctx *ImportContext, result *ImportResult) (err error) { return } -func remove_add_cart_cut(ctx *ImportContext, res *ImportResult) (err error) { - if err = remove_cart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { +func add_cart_cut(ctx *ImportContext, res *ImportResult) (err error) { + if err = add_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } - if err = add_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = add_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return remove_cart(ctx, &ImportResult{ResponseCode: http.StatusOK}) + } + return +} + +func remove_add_cart_cut(ctx *ImportContext, res *ImportResult) (err error) { + if err = remove_cart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { return } - return add_cut(ctx, res) + return add_cart_cut(ctx, res) } func is_cart_member_of_show(ctx *ImportContext, res *ImportResult, carts []uint) (found bool) { @@ -484,7 +548,10 @@ func add_show_cart_cut(ctx *ImportContext, res *ImportResult, carts []uint) (err } for _, cart := range carts { if cart == ctx.Cart { - return add_cut(ctx, res) + if err = add_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return remove_cart(ctx, &ImportResult{ResponseCode: http.StatusOK}) + } + return } } if err = remove_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { @@ -516,6 +583,8 @@ func cleanup_files(ctx *ImportContext) { func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { defer cleanup_files(ctx) + rhdl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) + if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(2, "importing", 0.0, ctx.ProgressCallBackData) } @@ -526,6 +595,8 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } } + rmCartOnErr := false + rmCutOnErr := false res = &ImportResult{ResponseCode: http.StatusOK} if ctx.ShowId != 0 { var show_carts []uint @@ -541,6 +612,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { if err = add_show_cart_cut(ctx, res, show_carts); err != nil || res.ResponseCode != http.StatusOK { return } + rmCartOnErr = true } else if ctx.GroupName != "" { if err = ctx.getMusicInfo(); err != nil { return @@ -548,12 +620,25 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } - } else if ctx.Cart != 0 && ctx.Cut == 0 { // TODO: we should add a DeleteCart option... - if err = ctx.getGroupOfCart(); err != nil { - return - } - if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return + rmCartOnErr = true + } else if ctx.Cart != 0 && ctx.Cut == 0 { + if ctx.ClearCart { + if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + rmCartOnErr = true + } else { + if err = add_cut(ctx, res); err != nil { + return + } + if res.ResponseCode != http.StatusOK { + if err = add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + rmCartOnErr = true + } else { + rmCutOnErr = true + } } } @@ -561,13 +646,22 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { if err = import_audio(ctx, res); err != nil { return } - res.Cart = ctx.Cart - res.Cut = ctx.Cut + if res.ResponseCode != http.StatusOK { + rmres := ImportResult{ResponseCode: http.StatusOK} + if rmCartOnErr { + if err = remove_cart(ctx, &rmres); err != nil { + return + } + } else if rmCutOnErr { + if err = remove_cut(ctx, &rmres); err != nil { + return + } + } + } } else { res.ResponseCode = http.StatusBadRequest res.ErrorString = "The request doesn't contain enough information to be processed" } - rhdl.Printf("ImportResult: %+v\n", res) return } diff --git a/rddb.go b/rddb.go index 34019e8..28fa6bd 100644 --- a/rddb.go +++ b/rddb.go @@ -33,7 +33,7 @@ import ( ) var ( - showMacroRe = regexp.MustCompile(`^LL 1 ([^ ]+) 0\!$`) + showMacroRe = regexp.MustCompile(`^LL 1 ([^ ]+) 0\!$`) mysqlTableNameRe = regexp.MustCompile(`^[_0-9a-zA-Z-]+$`) ) -- cgit v0.10.2 From d045588f048cfdb6bcdd36dcd1cdbe38318fd39a Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 18 Dec 2015 00:52:49 +0100 Subject: local:// sanitzes path now diff --git a/conf.go b/conf.go index 2386ba9..5f22782 100644 --- a/conf.go +++ b/conf.go @@ -43,6 +43,7 @@ type Config struct { db_user string db_passwd string db_db string + LocalFetchDir string ImportParamDefaults } @@ -67,7 +68,7 @@ func (self *Config) read_config_file() error { return nil } -func NewConfig(configfile, rdxport_endpoint, temp_dir *string) (conf *Config, err error) { +func NewConfig(configfile, rdxport_endpoint, temp_dir, local_fetch_dir *string) (conf *Config, err error) { conf = new(Config) conf.configfile = *configfile if err = conf.read_config_file(); err != nil { @@ -75,6 +76,7 @@ func NewConfig(configfile, rdxport_endpoint, temp_dir *string) (conf *Config, er } conf.RDXportEndpoint = *rdxport_endpoint conf.TempDir = *temp_dir + conf.LocalFetchDir = *local_fetch_dir conf.ImportParamDefaults.Channels = 2 conf.ImportParamDefaults.NormalizationLevel = -12 conf.ImportParamDefaults.AutotrimLevel = 0 diff --git a/fetcher.go b/fetcher.go index 38e1d7f..fed51bf 100644 --- a/fetcher.go +++ b/fetcher.go @@ -32,6 +32,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strings" ) @@ -129,19 +130,18 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { return } -// TODO: check path to import from -> don't touch problematic files like /etc/shadow... -// the daemon shouldn't be running as a user who can do any harm anyway -// still: let's make a special configurable directory the local:/// dir -// and only allow absolute paths here which will be based on the -// 'local' directory -// TODO: also check if file exists and is accessable!!! otherwise curl will blow up -// with a not-easy-to-understand error func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData) } - ctx.SourceFile = uri.Path + + ctx.SourceFile = filepath.Join(ctx.Config.LocalFetchDir, path.Clean("/"+uri.Path)) + var src *os.File + if src, err = os.Open(ctx.SourceFile); err != nil { + return + } + defer src.Close() ctx.DeleteSourceFile = false ctx.DeleteSourceDir = false return @@ -182,7 +182,6 @@ func fetcher_init() { } } -// TODO: make sure a (partially) fetched file get's deleted on error func FetchFile(ctx *ImportContext) (err error) { var uri *url.URL -- cgit v0.10.2 From 21027f52089faf514ecd2c5c1e8bfa4c5f8fe8d3 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 18 Dec 2015 03:36:46 +0100 Subject: added basic version of a session store diff --git a/session.go b/session.go new file mode 100644 index 0000000..37a4b54 --- /dev/null +++ b/session.go @@ -0,0 +1,265 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "encoding/base32" + "fmt" + "github.com/satori/go.uuid" + "strings" +) + +type Session struct { + ImportContext + ImportResult + // TODO: add creation time for timeout +} + +type ProgressData struct { + step int + step_name string + progress float64 + userdata interface{} +} + +type newSessionResponse struct { + id string + err error +} + +type newSessionRequest struct { + ctx ImportContext + response chan newSessionResponse +} + +type sessionRunResponse struct { + err error +} + +type sessionRunRequest struct { + user string + id string + response chan sessionRunResponse +} + +type sessionAddProgressHandlerResponse struct { + err error +} + +type sessionAddProgressHandlerRequest struct { + user string + id string + userdata interface{} + handler chan ProgressData + response chan sessionAddProgressHandlerResponse +} + +type sessionAddDoneHandlerResponse struct { + err error +} + +type sessionAddDoneHandlerRequest struct { + user string + id string + userdata interface{} + handler chan ImportResult + response chan sessionAddDoneHandlerResponse +} + +type sessionRemoveResponse struct { + err error +} + +type sessionRemoveRequest struct { + user string + id string + response chan sessionRemoveResponse +} + +type SessionStore struct { + store map[string]map[string]Session + quit chan bool + done chan bool + newChan chan newSessionRequest + runChan chan sessionRunRequest + addProgressChan chan sessionAddProgressHandlerRequest + addDoneChan chan sessionAddDoneHandlerRequest + removeChan chan sessionRemoveRequest +} + +func (self *SessionStore) Init() (err error) { + return +} + +func (self *SessionStore) newSession(ctx ImportContext) (resp newSessionResponse) { + b := uuid.NewV4().Bytes() + resp.id = strings.ToLower(strings.TrimRight(base32.StdEncoding.EncodeToString(b), "=")) + if _, exists := self.store[ctx.UserName]; !exists { + self.store[ctx.UserName] = make(map[string]Session) + } + self.store[ctx.UserName][resp.id] = Session{ImportContext: ctx} + rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) + return +} + +func (self *SessionStore) NewSession(ctx ImportContext) (string, error) { + req := newSessionRequest{} + req.ctx = ctx + req.response = make(chan newSessionResponse) + self.newChan <- req + + res := <-req.response + if res.err != nil { + return "", res.err + } + return res.id, nil +} + +func (self *SessionStore) run(user, id string) (resp sessionRunResponse) { + rhdl.Printf("SessionStore: running session '%s/%s'", user, id) + return +} + +func (self *SessionStore) Run(user, id string) error { + req := sessionRunRequest{} + req.user = user + req.id = id + req.response = make(chan sessionRunResponse) + self.runChan <- req + + res := <-req.response + return res.err +} + +func (self *SessionStore) addProgressHandler(user, id string, userdata interface{}, handler chan ProgressData) (resp sessionAddProgressHandlerResponse) { + rhdl.Printf("SessionStore: adding progress handler to '%s/%s'", user, id) + return +} + +func (self *SessionStore) AddProgressHandler(user, id string, userdata interface{}, handler chan ProgressData) error { + req := sessionAddProgressHandlerRequest{} + req.user = user + req.id = id + req.userdata = userdata + req.handler = handler + req.response = make(chan sessionAddProgressHandlerResponse) + self.addProgressChan <- req + + res := <-req.response + return res.err +} + +func (self *SessionStore) addDoneHandler(user, id string, userdata interface{}, handler chan ImportResult) (resp sessionAddDoneHandlerResponse) { + rhdl.Printf("SessionStore: adding done handler to '%s/%s'", user, id) + return +} + +func (self *SessionStore) AddDoneHandler(user, id string, userdata interface{}, handler chan ImportResult) error { + req := sessionAddDoneHandlerRequest{} + req.user = user + req.id = id + req.userdata = userdata + req.handler = handler + req.response = make(chan sessionAddDoneHandlerResponse) + self.addDoneChan <- req + + res := <-req.response + return res.err +} + +func (self *SessionStore) remove(user, id string) (resp sessionRemoveResponse) { + if _, exists := self.store[user][id]; exists { + delete(self.store[user], id) + rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) + if _, exists := self.store[user]; exists { + if len(self.store[user]) == 0 { + delete(self.store, user) + rhdl.Printf("SessionStore: removed user '%s'", user) + } + } + } else { + resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + } + return +} + +func (self *SessionStore) Remove(user, id string) error { + req := sessionRemoveRequest{} + req.user = user + req.id = id + req.response = make(chan sessionRemoveResponse) + self.removeChan <- req + + res := <-req.response + return res.err +} + +func (self *SessionStore) dispatchRequests() { + defer func() { self.done <- true }() + for { + select { + case <-self.quit: + return + case req := <-self.newChan: + req.response <- self.newSession(req.ctx) + case req := <-self.runChan: + req.response <- self.run(req.user, req.id) + case req := <-self.addProgressChan: + req.response <- self.addProgressHandler(req.user, req.id, req.userdata, req.handler) + case req := <-self.addDoneChan: + req.response <- self.addDoneHandler(req.user, req.id, req.userdata, req.handler) + case req := <-self.removeChan: + req.response <- self.remove(req.user, req.id) + } + } +} + +func (self *SessionStore) Cleanup() { + self.quit <- true + <-self.done + close(self.quit) + close(self.done) + close(self.newChan) + close(self.runChan) + close(self.addProgressChan) + close(self.addDoneChan) + close(self.removeChan) +} + +func NewSessionStore(conf *Config) (store *SessionStore, err error) { + store = new(SessionStore) + + store.quit = make(chan bool) + store.done = make(chan bool) + store.store = make(map[string]map[string]Session) + store.newChan = make(chan newSessionRequest) + store.runChan = make(chan sessionRunRequest) + store.addProgressChan = make(chan sessionAddProgressHandlerRequest) + store.addDoneChan = make(chan sessionAddDoneHandlerRequest) + store.removeChan = make(chan sessionRemoveRequest) + + go store.dispatchRequests() + return +} -- cgit v0.10.2 From 16f3a42d4872dfa3d167ccf760f91d624eee312c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 18 Dec 2015 03:40:42 +0100 Subject: added check for existence of requested session diff --git a/session.go b/session.go index 37a4b54..219c233 100644 --- a/session.go +++ b/session.go @@ -138,7 +138,11 @@ func (self *SessionStore) NewSession(ctx ImportContext) (string, error) { } func (self *SessionStore) run(user, id string) (resp sessionRunResponse) { - rhdl.Printf("SessionStore: running session '%s/%s'", user, id) + if _, exists := self.store[user][id]; exists { + rhdl.Printf("SessionStore: running session '%s/%s'", user, id) + } else { + resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + } return } @@ -154,7 +158,11 @@ func (self *SessionStore) Run(user, id string) error { } func (self *SessionStore) addProgressHandler(user, id string, userdata interface{}, handler chan ProgressData) (resp sessionAddProgressHandlerResponse) { - rhdl.Printf("SessionStore: adding progress handler to '%s/%s'", user, id) + if _, exists := self.store[user][id]; exists { + rhdl.Printf("SessionStore: adding progress handler to '%s/%s'", user, id) + } else { + resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + } return } @@ -172,7 +180,11 @@ func (self *SessionStore) AddProgressHandler(user, id string, userdata interface } func (self *SessionStore) addDoneHandler(user, id string, userdata interface{}, handler chan ImportResult) (resp sessionAddDoneHandlerResponse) { - rhdl.Printf("SessionStore: adding done handler to '%s/%s'", user, id) + if _, exists := self.store[user][id]; exists { + rhdl.Printf("SessionStore: adding done handler to '%s/%s'", user, id) + } else { + resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + } return } -- cgit v0.10.2 From d1ea14e9be0891f7e19bdf5090a70aad9caa45ba Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 19 Dec 2015 03:55:57 +0100 Subject: improved show based imports diff --git a/importer.go b/importer.go index 94065f6..afe422f 100644 --- a/importer.go +++ b/importer.go @@ -525,6 +525,7 @@ func is_cart_member_of_show(ctx *ImportContext, res *ImportResult, carts []uint) } res.ResponseCode = http.StatusBadRequest res.ErrorString = fmt.Sprintf("Requested cart %d is not a member of show: %d", ctx.Cart, ctx.ShowId) + res.Cart = ctx.Cart return false } @@ -609,8 +610,14 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { if err = clear_show_carts(ctx, res, show_carts); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { return } - if err = add_show_cart_cut(ctx, res, show_carts); err != nil || res.ResponseCode != http.StatusOK { - return + if ctx.ClearCart && !ctx.ClearShowCarts { + if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + } else { + if err = add_show_cart_cut(ctx, res, show_carts); err != nil || res.ResponseCode != http.StatusOK { + return + } } rmCartOnErr = true } else if ctx.GroupName != "" { -- cgit v0.10.2 From e56c01825d1c6366b743d7e4e3e10d6189c01c7a Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 19 Dec 2015 21:07:52 +0100 Subject: tested music pool import, cleaned up help output for telnet interface diff --git a/importer.go b/importer.go index afe422f..1404259 100644 --- a/importer.go +++ b/importer.go @@ -106,6 +106,9 @@ func (ctx *ImportContext) SanityCheck() error { if !ismusic { return fmt.Errorf("supplied GroupName is not a music pool") } + if ctx.Cart != 0 || ctx.Cut != 0 { + return fmt.Errorf("Cart and Cut must not be supplied when importing into a music group") + } return nil } if ctx.Cart == 0 { @@ -193,6 +196,8 @@ func (ctx *ImportContext) getMusicInfo() (err error) { ctx.AutotrimLevel = res.trim_lvl ctx.Channels = 2 ctx.UseMetaData = true + ctx.Cart = 0 + ctx.Cut = 0 return } @@ -624,7 +629,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { if err = ctx.getMusicInfo(); err != nil { return } - if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } rmCartOnErr = true diff --git a/rddb.go b/rddb.go index 28fa6bd..25ef876 100644 --- a/rddb.go +++ b/rddb.go @@ -247,7 +247,7 @@ func (self *RdDb) getShowInfo(showid uint) (result getShowInfoResult) { func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { var cnt int - if result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&cnt); result.err != nil { + if result.err = self.checkMusicGroupStmt.QueryRow(group).Scan(&cnt); result.err != nil { if result.err == sql.ErrNoRows { result.err = nil result.ismusic = false @@ -259,7 +259,7 @@ func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { } func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { - result.err = self.checkMusicGroupStmt.QueryRow(group).Scan(&result.norm_lvl, &result.trim_lvl) + result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&result.norm_lvl, &result.trim_lvl) if result.err != nil { if result.err == sql.ErrNoRows { result.err = fmt.Errorf("music pool '%s' not found", group) -- cgit v0.10.2 From 27e7ed5a42b053bbb8a3b902550fbe6e9eca39c9 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 19 Dec 2015 23:07:08 +0100 Subject: fetcher now checks for permissions improved error handling for fetcher diff --git a/fetcher.go b/fetcher.go index fed51bf..300abef 100644 --- a/fetcher.go +++ b/fetcher.go @@ -29,6 +29,7 @@ import ( "github.com/golang-basic/go-curl" "io/ioutil" "mime" + "net/http" "net/url" "os" "path" @@ -36,6 +37,11 @@ import ( "strings" ) +type FetchResult struct { + ResponseCode int + ErrorString string +} + type FetcherCurlCBData struct { basepath string filename string @@ -83,7 +89,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { return true } -func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { +func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) easy := curl.EasyInit() @@ -116,7 +122,7 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { - err = fmt.Errorf("fetcher('%s'): %s", ctx.SourceUri, err) + err = fmt.Errorf("curl-fetcher('%s'): %s", ctx.SourceUri, err) return } @@ -130,7 +136,7 @@ func FetchFileCurl(ctx *ImportContext, uri *url.URL) (err error) { return } -func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { +func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData) @@ -139,15 +145,17 @@ func FetchFileLocal(ctx *ImportContext, uri *url.URL) (err error) { ctx.SourceFile = filepath.Join(ctx.Config.LocalFetchDir, path.Clean("/"+uri.Path)) var src *os.File if src, err = os.Open(ctx.SourceFile); err != nil { - return + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("local-file open(): %s", err) + return nil } - defer src.Close() + src.Close() ctx.DeleteSourceFile = false ctx.DeleteSourceDir = false return } -type FetchFunc func(*ImportContext, *url.URL) (err error) +type FetchFunc func(*ImportContext, *FetchResult, *url.URL) (err error) // TODO: implement fetchers for: // archiv:// @@ -182,15 +190,52 @@ func fetcher_init() { } } -func FetchFile(ctx *ImportContext) (err error) { +func checkPassword(ctx *ImportContext, result *FetchResult) (err error) { + cached := true + + for { + req := getPasswordRequest{} + req.username = ctx.UserName + req.cached = cached + req.response = make(chan getPasswordResult) + ctx.RdDb.getPasswordChan <- req + + res := <-req.response + if res.err != nil { + return res.err + } + if ctx.Password == res.password { + return nil + } + if cached { + cached = false + } else { + break + } + } + result.ResponseCode = http.StatusUnauthorized + result.ErrorString = "invalid username and/or password" + return +} + +func FetchFile(ctx *ImportContext) (res *FetchResult, err error) { + res = &FetchResult{ResponseCode: http.StatusOK} var uri *url.URL if uri, err = url.Parse(ctx.SourceUri); err != nil { - return + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("parsing uri: %s", err) + return res, nil + } + + if !ctx.Trusted { + if err = checkPassword(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } } if fetcher, ok := fetchers[uri.Scheme]; ok { - err = fetcher(ctx, uri) + err = fetcher(ctx, res, uri) } else { err = fmt.Errorf("No fetcher for uri scheme '%s' found.", uri.Scheme) } -- cgit v0.10.2 From 7a014c8f0858703ea4f70d1201976ae8c2b45d73 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 20 Dec 2015 02:29:13 +0100 Subject: improved type safety diff --git a/fetcher.go b/fetcher.go index 300abef..a8ac22c 100644 --- a/fetcher.go +++ b/fetcher.go @@ -101,7 +101,7 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro cbdata := &FetcherCurlCBData{remotename: path.Base(uri.Path)} defer cbdata.Cleanup() - if cbdata.basepath, err = ioutil.TempDir(ctx.Config.TempDir, "rhimportd-"); err != nil { + if cbdata.basepath, err = ioutil.TempDir(ctx.conf.TempDir, "rhimportd-"); err != nil { return } @@ -142,7 +142,7 @@ func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err err ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData) } - ctx.SourceFile = filepath.Join(ctx.Config.LocalFetchDir, path.Clean("/"+uri.Path)) + ctx.SourceFile = filepath.Join(ctx.conf.LocalFetchDir, path.Clean("/"+uri.Path)) var src *os.File if src, err = os.Open(ctx.SourceFile); err != nil { res.ResponseCode = http.StatusBadRequest @@ -194,13 +194,14 @@ func checkPassword(ctx *ImportContext, result *FetchResult) (err error) { cached := true for { + res_ch := make(chan getPasswordResult) req := getPasswordRequest{} req.username = ctx.UserName req.cached = cached - req.response = make(chan getPasswordResult) - ctx.RdDb.getPasswordChan <- req + req.response = res_ch + ctx.rddb.getPasswordChan <- req - res := <-req.response + res := <-res_ch if res.err != nil { return res.err } diff --git a/importer.go b/importer.go index 1404259..f65515b 100644 --- a/importer.go +++ b/importer.go @@ -40,8 +40,8 @@ var ( ) type ImportContext struct { - *Config - *RdDb + conf *Config + rddb *RdDbChan UserName string Password string Trusted bool @@ -63,10 +63,10 @@ type ImportContext struct { ProgressCallBackData interface{} } -func NewImportContext(conf *Config, rddb *RdDb, user string) *ImportContext { +func NewImportContext(conf *Config, rddb *RdDbChan, user string) *ImportContext { ctx := new(ImportContext) - ctx.Config = conf - ctx.RdDb = rddb + ctx.conf = conf + ctx.rddb = rddb ctx.UserName = user ctx.Password = "" ctx.Trusted = false @@ -121,13 +121,14 @@ func (ctx *ImportContext) SanityCheck() error { } func (ctx *ImportContext) getPassword(cached bool) (err error) { + res_ch := make(chan getPasswordResult) req := getPasswordRequest{} req.username = ctx.UserName req.cached = cached - req.response = make(chan getPasswordResult) - ctx.RdDb.getPasswordChan <- req + req.response = res_ch + ctx.rddb.getPasswordChan <- req - res := <-req.response + res := <-res_ch if res.err != nil { return res.err } @@ -136,12 +137,13 @@ func (ctx *ImportContext) getPassword(cached bool) (err error) { } func (ctx *ImportContext) getGroupOfCart() error { + res_ch := make(chan getGroupOfCartResult) req := getGroupOfCartRequest{} req.cart = ctx.Cart - req.response = make(chan getGroupOfCartResult) - ctx.RdDb.getGroupOfCartChan <- req + req.response = res_ch + ctx.rddb.getGroupOfCartChan <- req - res := <-req.response + res := <-res_ch if res.err != nil { return res.err } @@ -150,12 +152,13 @@ func (ctx *ImportContext) getGroupOfCart() error { } func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { + res_ch := make(chan getShowInfoResult) req := getShowInfoRequest{} req.showid = ctx.ShowId - req.response = make(chan getShowInfoResult) - ctx.RdDb.getShowInfoChan <- req + req.response = res_ch + ctx.rddb.getShowInfoChan <- req - res := <-req.response + res := <-res_ch if res.err != nil { err = res.err return @@ -170,12 +173,13 @@ func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { } func (ctx *ImportContext) checkMusicGroup() (bool, error) { + res_ch := make(chan checkMusicGroupResult) req := checkMusicGroupRequest{} req.group = ctx.GroupName - req.response = make(chan checkMusicGroupResult) - ctx.RdDb.checkMusicGroupChan <- req + req.response = res_ch + ctx.rddb.checkMusicGroupChan <- req - res := <-req.response + res := <-res_ch if res.err != nil { return false, res.err } @@ -183,12 +187,13 @@ func (ctx *ImportContext) checkMusicGroup() (bool, error) { } func (ctx *ImportContext) getMusicInfo() (err error) { + res_ch := make(chan getMusicInfoResult) req := getMusicInfoRequest{} req.group = ctx.GroupName - req.response = make(chan getMusicInfoResult) - ctx.RdDb.getMusicInfoChan <- req + req.response = res_ch + ctx.rddb.getMusicInfoChan <- req - res := <-req.response + res := <-res_ch if res.err != nil { return res.err } @@ -251,7 +256,7 @@ func add_cart(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -296,7 +301,7 @@ func add_cut(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -343,7 +348,7 @@ func remove_cart(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -380,7 +385,7 @@ func remove_cut(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.Config.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -455,7 +460,7 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { if easy != nil { defer easy.Cleanup() - easy.Setopt(curl.OPT_URL, ctx.Config.RDXportEndpoint) + easy.Setopt(curl.OPT_URL, ctx.conf.RDXportEndpoint) easy.Setopt(curl.OPT_POST, true) var form *curl.Form diff --git a/rddb.go b/rddb.go index 25ef876..f596171 100644 --- a/rddb.go +++ b/rddb.go @@ -49,7 +49,7 @@ type getPasswordResult struct { type getPasswordRequest struct { username string cached bool - response chan getPasswordResult + response chan<- getPasswordResult } type getGroupOfCartResult struct { @@ -59,7 +59,7 @@ type getGroupOfCartResult struct { type getGroupOfCartRequest struct { cart uint - response chan getGroupOfCartResult + response chan<- getGroupOfCartResult } type getShowInfoResult struct { @@ -73,7 +73,7 @@ type getShowInfoResult struct { type getShowInfoRequest struct { showid uint - response chan getShowInfoResult + response chan<- getShowInfoResult } type checkMusicGroupResult struct { @@ -83,7 +83,7 @@ type checkMusicGroupResult struct { type checkMusicGroupRequest struct { group string - response chan checkMusicGroupResult + response chan<- checkMusicGroupResult } type getMusicInfoResult struct { @@ -94,7 +94,15 @@ type getMusicInfoResult struct { type getMusicInfoRequest struct { group string - response chan getMusicInfoResult + response chan<- getMusicInfoResult +} + +type RdDbChan struct { + getPasswordChan chan<- getPasswordRequest + getGroupOfCartChan chan<- getGroupOfCartRequest + getShowInfoChan chan<- getShowInfoRequest + checkMusicGroupChan chan<- checkMusicGroupRequest + getMusicInfoChan chan<- getMusicInfoRequest } type RdDb struct { @@ -289,6 +297,16 @@ func (self *RdDb) dispatchRequests() { } } +func (self *RdDb) GetChannels() *RdDbChan { + ch := &RdDbChan{} + ch.getPasswordChan = self.getPasswordChan + ch.getGroupOfCartChan = self.getGroupOfCartChan + ch.getShowInfoChan = self.getShowInfoChan + ch.checkMusicGroupChan = self.checkMusicGroupChan + ch.getMusicInfoChan = self.getMusicInfoChan + return ch +} + func (self *RdDb) Cleanup() { self.quit <- true <-self.done diff --git a/session.go b/session.go index 219c233..718d13e 100644 --- a/session.go +++ b/session.go @@ -61,7 +61,7 @@ type sessionRunResponse struct { type sessionRunRequest struct { user string id string - response chan sessionRunResponse + response chan<- sessionRunResponse } type sessionAddProgressHandlerResponse struct { @@ -72,8 +72,8 @@ type sessionAddProgressHandlerRequest struct { user string id string userdata interface{} - handler chan ProgressData - response chan sessionAddProgressHandlerResponse + handler chan<- ProgressData + response chan<- sessionAddProgressHandlerResponse } type sessionAddDoneHandlerResponse struct { @@ -84,8 +84,8 @@ type sessionAddDoneHandlerRequest struct { user string id string userdata interface{} - handler chan ImportResult - response chan sessionAddDoneHandlerResponse + handler chan<- ImportResult + response chan<- sessionAddDoneHandlerResponse } type sessionRemoveResponse struct { @@ -125,12 +125,13 @@ func (self *SessionStore) newSession(ctx ImportContext) (resp newSessionResponse } func (self *SessionStore) NewSession(ctx ImportContext) (string, error) { + res_ch := make(chan newSessionResponse) req := newSessionRequest{} req.ctx = ctx - req.response = make(chan newSessionResponse) + req.response = res_ch self.newChan <- req - res := <-req.response + res := <-res_ch if res.err != nil { return "", res.err } @@ -147,17 +148,18 @@ func (self *SessionStore) run(user, id string) (resp sessionRunResponse) { } func (self *SessionStore) Run(user, id string) error { + res_ch := make(chan sessionRunResponse) req := sessionRunRequest{} req.user = user req.id = id - req.response = make(chan sessionRunResponse) + req.response = res_ch self.runChan <- req - res := <-req.response + res := <-res_ch return res.err } -func (self *SessionStore) addProgressHandler(user, id string, userdata interface{}, handler chan ProgressData) (resp sessionAddProgressHandlerResponse) { +func (self *SessionStore) addProgressHandler(user, id string, userdata interface{}, handler chan<- ProgressData) (resp sessionAddProgressHandlerResponse) { if _, exists := self.store[user][id]; exists { rhdl.Printf("SessionStore: adding progress handler to '%s/%s'", user, id) } else { @@ -166,20 +168,21 @@ func (self *SessionStore) addProgressHandler(user, id string, userdata interface return } -func (self *SessionStore) AddProgressHandler(user, id string, userdata interface{}, handler chan ProgressData) error { +func (self *SessionStore) AddProgressHandler(user, id string, userdata interface{}, handler chan<- ProgressData) error { + res_ch := make(chan sessionAddProgressHandlerResponse) req := sessionAddProgressHandlerRequest{} req.user = user req.id = id req.userdata = userdata req.handler = handler - req.response = make(chan sessionAddProgressHandlerResponse) + req.response = res_ch self.addProgressChan <- req - res := <-req.response + res := <-res_ch return res.err } -func (self *SessionStore) addDoneHandler(user, id string, userdata interface{}, handler chan ImportResult) (resp sessionAddDoneHandlerResponse) { +func (self *SessionStore) addDoneHandler(user, id string, userdata interface{}, handler chan<- ImportResult) (resp sessionAddDoneHandlerResponse) { if _, exists := self.store[user][id]; exists { rhdl.Printf("SessionStore: adding done handler to '%s/%s'", user, id) } else { @@ -188,16 +191,17 @@ func (self *SessionStore) addDoneHandler(user, id string, userdata interface{}, return } -func (self *SessionStore) AddDoneHandler(user, id string, userdata interface{}, handler chan ImportResult) error { +func (self *SessionStore) AddDoneHandler(user, id string, userdata interface{}, handler chan<- ImportResult) error { + res_ch := make(chan sessionAddDoneHandlerResponse) req := sessionAddDoneHandlerRequest{} req.user = user req.id = id req.userdata = userdata req.handler = handler - req.response = make(chan sessionAddDoneHandlerResponse) + req.response = res_ch self.addDoneChan <- req - res := <-req.response + res := <-res_ch return res.err } @@ -218,13 +222,14 @@ func (self *SessionStore) remove(user, id string) (resp sessionRemoveResponse) { } func (self *SessionStore) Remove(user, id string) error { + res_ch := make(chan sessionRemoveResponse) req := sessionRemoveRequest{} req.user = user req.id = id - req.response = make(chan sessionRemoveResponse) + req.response = res_ch self.removeChan <- req - res := <-req.response + res := <-res_ch return res.err } -- cgit v0.10.2 From 795d37b3aa2ee04f9a56a9a75284b0a4ac8a4b0f Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 20 Dec 2015 05:46:01 +0100 Subject: session store and session kind of work now still a lot to be done and a lot of testing is needed! diff --git a/importer.go b/importer.go index f65515b..292d376 100644 --- a/importer.go +++ b/importer.go @@ -39,6 +39,8 @@ var ( bool2str = map[bool]string{false: "0", true: "1"} ) +type ImportProgressCB func(step int, step_name string, progress float64, userdata interface{}) + type ImportContext struct { conf *Config rddb *RdDbChan @@ -59,7 +61,7 @@ type ImportContext struct { SourceFile string DeleteSourceFile bool DeleteSourceDir bool - ProgressCallBack func(step int, step_name string, progress float64, userdata interface{}) + ProgressCallBack ImportProgressCB ProgressCallBackData interface{} } diff --git a/rddb.go b/rddb.go index f596171..bb481f4 100644 --- a/rddb.go +++ b/rddb.go @@ -297,7 +297,7 @@ func (self *RdDb) dispatchRequests() { } } -func (self *RdDb) GetChannels() *RdDbChan { +func (self *RdDb) GetInterface() *RdDbChan { ch := &RdDbChan{} ch.getPasswordChan = self.getPasswordChan ch.getGroupOfCartChan = self.getGroupOfCartChan diff --git a/session.go b/session.go index 718d13e..ce84053 100644 --- a/session.go +++ b/session.go @@ -25,43 +25,33 @@ package rhimport import ( - "encoding/base32" - "fmt" - "github.com/satori/go.uuid" - "strings" + "time" ) +type SessionChan struct { + runChan chan<- bool + cancelChan chan<- bool + addProgressChan chan<- sessionAddProgressHandlerRequest + addDoneChan chan<- sessionAddDoneHandlerRequest +} + type Session struct { - ImportContext - ImportResult - // TODO: add creation time for timeout + ctx ImportContext + running bool + done chan bool + progressIntChan chan ProgressData + doneIntChan chan ImportResult + runChan chan bool + cancelChan chan bool + addProgressChan chan sessionAddProgressHandlerRequest + addDoneChan chan sessionAddDoneHandlerRequest + // TODO: add pub/sub for progress and done } type ProgressData struct { step int step_name string progress float64 - userdata interface{} -} - -type newSessionResponse struct { - id string - err error -} - -type newSessionRequest struct { - ctx ImportContext - response chan newSessionResponse -} - -type sessionRunResponse struct { - err error -} - -type sessionRunRequest struct { - user string - id string - response chan<- sessionRunResponse } type sessionAddProgressHandlerResponse struct { @@ -69,10 +59,8 @@ type sessionAddProgressHandlerResponse struct { } type sessionAddProgressHandlerRequest struct { - user string - id string userdata interface{} - handler chan<- ProgressData + callback ImportProgressCB response chan<- sessionAddProgressHandlerResponse } @@ -81,100 +69,57 @@ type sessionAddDoneHandlerResponse struct { } type sessionAddDoneHandlerRequest struct { - user string - id string userdata interface{} - handler chan<- ImportResult + callback func(*ImportResult, interface{}) response chan<- sessionAddDoneHandlerResponse } -type sessionRemoveResponse struct { - err error -} - -type sessionRemoveRequest struct { - user string - id string - response chan sessionRemoveResponse -} - -type SessionStore struct { - store map[string]map[string]Session - quit chan bool - done chan bool - newChan chan newSessionRequest - runChan chan sessionRunRequest - addProgressChan chan sessionAddProgressHandlerRequest - addDoneChan chan sessionAddDoneHandlerRequest - removeChan chan sessionRemoveRequest +func session_progress_callback(step int, step_name string, progress float64, userdata interface{}) { + out := userdata.(chan<- ProgressData) + out <- ProgressData{step, step_name, progress} } -func (self *SessionStore) Init() (err error) { - return -} - -func (self *SessionStore) newSession(ctx ImportContext) (resp newSessionResponse) { - b := uuid.NewV4().Bytes() - resp.id = strings.ToLower(strings.TrimRight(base32.StdEncoding.EncodeToString(b), "=")) - if _, exists := self.store[ctx.UserName]; !exists { - self.store[ctx.UserName] = make(map[string]Session) +// TODO: actually call import here +func session_import_run(ctx ImportContext, done chan<- ImportResult) { + rhdl.Printf("faking import for: %+v", ctx) + for i := 0; i < 100; i++ { + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(42, "faking", float64(i)/100.0, ctx.ProgressCallBackData) + } + time.Sleep(100 * time.Millisecond) } - self.store[ctx.UserName][resp.id] = Session{ImportContext: ctx} - rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) - return -} - -func (self *SessionStore) NewSession(ctx ImportContext) (string, error) { - res_ch := make(chan newSessionResponse) - req := newSessionRequest{} - req.ctx = ctx - req.response = res_ch - self.newChan <- req - - res := <-res_ch - if res.err != nil { - return "", res.err + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(42, "faking", 1.0, ctx.ProgressCallBackData) } - return res.id, nil + done <- ImportResult{200, "OK", 0, 0} } -func (self *SessionStore) run(user, id string) (resp sessionRunResponse) { - if _, exists := self.store[user][id]; exists { - rhdl.Printf("SessionStore: running session '%s/%s'", user, id) - } else { - resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) - } +func (self *Session) run() { + self.ctx.ProgressCallBack = session_progress_callback + self.ctx.ProgressCallBackData = (chan<- ProgressData)(self.progressIntChan) + go session_import_run(self.ctx, self.doneIntChan) + self.running = true return } -func (self *SessionStore) Run(user, id string) error { - res_ch := make(chan sessionRunResponse) - req := sessionRunRequest{} - req.user = user - req.id = id - req.response = res_ch - self.runChan <- req +func (self *SessionChan) Run() { + self.runChan <- true +} - res := <-res_ch - return res.err +func (self *SessionChan) Cancel() { + self.cancelChan <- true } -func (self *SessionStore) addProgressHandler(user, id string, userdata interface{}, handler chan<- ProgressData) (resp sessionAddProgressHandlerResponse) { - if _, exists := self.store[user][id]; exists { - rhdl.Printf("SessionStore: adding progress handler to '%s/%s'", user, id) - } else { - resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) - } +func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressCB) (resp sessionAddProgressHandlerResponse) { + rhdl.Printf("Session: addProgressHandler called with: %+v", userdata) return } -func (self *SessionStore) AddProgressHandler(user, id string, userdata interface{}, handler chan<- ProgressData) error { +func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgressCB) error { res_ch := make(chan sessionAddProgressHandlerResponse) req := sessionAddProgressHandlerRequest{} - req.user = user - req.id = id req.userdata = userdata - req.handler = handler + req.callback = cb req.response = res_ch self.addProgressChan <- req @@ -182,22 +127,16 @@ func (self *SessionStore) AddProgressHandler(user, id string, userdata interface return res.err } -func (self *SessionStore) addDoneHandler(user, id string, userdata interface{}, handler chan<- ImportResult) (resp sessionAddDoneHandlerResponse) { - if _, exists := self.store[user][id]; exists { - rhdl.Printf("SessionStore: adding done handler to '%s/%s'", user, id) - } else { - resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) - } +func (self *Session) addDoneHandler(userdata interface{}, cb func(*ImportResult, interface{})) (resp sessionAddDoneHandlerResponse) { + rhdl.Printf("Session: addDoneHandler called with: %+v", userdata) return } -func (self *SessionStore) AddDoneHandler(user, id string, userdata interface{}, handler chan<- ImportResult) error { +func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(*ImportResult, interface{})) error { res_ch := make(chan sessionAddDoneHandlerResponse) req := sessionAddDoneHandlerRequest{} - req.user = user - req.id = id req.userdata = userdata - req.handler = handler + req.callback = cb req.response = res_ch self.addDoneChan <- req @@ -205,78 +144,69 @@ func (self *SessionStore) AddDoneHandler(user, id string, userdata interface{}, return res.err } -func (self *SessionStore) remove(user, id string) (resp sessionRemoveResponse) { - if _, exists := self.store[user][id]; exists { - delete(self.store[user], id) - rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) - if _, exists := self.store[user]; exists { - if len(self.store[user]) == 0 { - delete(self.store, user) - rhdl.Printf("SessionStore: removed user '%s'", user) - } - } - } else { - resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) - } - return -} - -func (self *SessionStore) Remove(user, id string) error { - res_ch := make(chan sessionRemoveResponse) - req := sessionRemoveRequest{} - req.user = user - req.id = id - req.response = res_ch - self.removeChan <- req - - res := <-res_ch - return res.err -} - -func (self *SessionStore) dispatchRequests() { +func (self *Session) dispatchRequests() { defer func() { self.done <- true }() for { select { - case <-self.quit: - return - case req := <-self.newChan: - req.response <- self.newSession(req.ctx) - case req := <-self.runChan: - req.response <- self.run(req.user, req.id) + case <-self.runChan: + self.run() + case <-self.cancelChan: + if self.running { + rhdl.Println("Session: canceling running imports is not yet implemented") + // TODO: send cancel to import goroutine once this got implemented + } else { + return + } case req := <-self.addProgressChan: - req.response <- self.addProgressHandler(req.user, req.id, req.userdata, req.handler) + req.response <- self.addProgressHandler(req.userdata, req.callback) case req := <-self.addDoneChan: - req.response <- self.addDoneHandler(req.user, req.id, req.userdata, req.handler) - case req := <-self.removeChan: - req.response <- self.remove(req.user, req.id) + req.response <- self.addDoneHandler(req.userdata, req.callback) + case progress := <-self.progressIntChan: + rhdl.Printf("Session: got progress: %+v", progress) + // TODO: call all subscribed progress handler + case result := <-self.doneIntChan: + self.running = false + rhdl.Printf("Session: import is done: %+v", result) + // TODO: call all subscribed done handler + // TODO: send remove request to session store? + return } } } -func (self *SessionStore) Cleanup() { - self.quit <- true +func (self *Session) getInterface() *SessionChan { + ch := &SessionChan{} + ch.runChan = self.runChan + ch.cancelChan = self.cancelChan + ch.addProgressChan = self.addProgressChan + ch.addDoneChan = self.addDoneChan + return ch +} + +func (self *Session) Cleanup() { + // TODO: this blocks if dispatchRequests has ended already... + self.cancelChan <- true <-self.done - close(self.quit) close(self.done) - close(self.newChan) + close(self.progressIntChan) + close(self.doneIntChan) close(self.runChan) + close(self.cancelChan) close(self.addProgressChan) close(self.addDoneChan) - close(self.removeChan) } -func NewSessionStore(conf *Config) (store *SessionStore, err error) { - store = new(SessionStore) - - store.quit = make(chan bool) - store.done = make(chan bool) - store.store = make(map[string]map[string]Session) - store.newChan = make(chan newSessionRequest) - store.runChan = make(chan sessionRunRequest) - store.addProgressChan = make(chan sessionAddProgressHandlerRequest) - store.addDoneChan = make(chan sessionAddDoneHandlerRequest) - store.removeChan = make(chan sessionRemoveRequest) - - go store.dispatchRequests() +func NewSession(ctx *ImportContext) (session *Session) { + session = new(Session) + session.running = false + session.ctx = *ctx + session.done = make(chan bool) + session.progressIntChan = make(chan ProgressData) + session.doneIntChan = make(chan ImportResult) + session.runChan = make(chan bool) + session.cancelChan = make(chan bool) + session.addProgressChan = make(chan sessionAddProgressHandlerRequest) + session.addDoneChan = make(chan sessionAddDoneHandlerRequest) + go session.dispatchRequests() return } diff --git a/session_store.go b/session_store.go new file mode 100644 index 0000000..b2e1538 --- /dev/null +++ b/session_store.go @@ -0,0 +1,206 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "encoding/base32" + "fmt" + "github.com/satori/go.uuid" + "strings" +) + +type newSessionResponse struct { + id string + session *SessionChan + err error +} + +type newSessionRequest struct { + ctx *ImportContext + response chan newSessionResponse +} + +type getSessionResponse struct { + session *SessionChan + err error +} + +type getSessionRequest struct { + user string + id string + response chan getSessionResponse +} + +type removeSessionResponse struct { + err error +} + +type removeSessionRequest struct { + user string + id string + response chan removeSessionResponse +} + +type SessionStoreChan struct { + newChan chan<- newSessionRequest + getChan chan<- getSessionRequest + removeChan chan<- removeSessionRequest +} + +type SessionStore struct { + store map[string]map[string]*Session + quit chan bool + done chan bool + newChan chan newSessionRequest + getChan chan getSessionRequest + removeChan chan removeSessionRequest +} + +func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { + b := uuid.NewV4().Bytes() + resp.id = strings.ToLower(strings.TrimRight(base32.StdEncoding.EncodeToString(b), "=")) + if _, exists := self.store[ctx.UserName]; !exists { + self.store[ctx.UserName] = make(map[string]*Session) + } + self.store[ctx.UserName][resp.id] = NewSession(ctx) + resp.session = self.store[ctx.UserName][resp.id].getInterface() + rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) + return +} + +func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, error) { + res_ch := make(chan newSessionResponse) + req := newSessionRequest{} + req.ctx = ctx + req.response = res_ch + self.newChan <- req + + res := <-res_ch + if res.err != nil { + return "", nil, res.err + } + return res.id, res.session, nil +} + +func (self *SessionStore) get(user, id string) (resp getSessionResponse) { + if session, exists := self.store[user][id]; exists { + resp.session = session.getInterface() + } else { + resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + } + return +} + +func (self *SessionStoreChan) Get(user, id string) (*SessionChan, error) { + res_ch := make(chan getSessionResponse) + req := getSessionRequest{} + req.user = user + req.id = id + req.response = res_ch + self.getChan <- req + + res := <-res_ch + if res.err != nil { + return nil, res.err + } + return res.session, nil +} + +func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { + if session, exists := self.store[user][id]; exists { + session.Cleanup() + delete(self.store[user], id) + rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) + if userstore, exists := self.store[user]; exists { + if len(userstore) == 0 { + delete(self.store, user) + rhdl.Printf("SessionStore: removed user '%s'", user) + } + } + } else { + resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + } + return +} + +func (self *SessionStoreChan) Remove(user, id string) error { + res_ch := make(chan removeSessionResponse) + req := removeSessionRequest{} + req.user = user + req.id = id + req.response = res_ch + self.removeChan <- req + + res := <-res_ch + return res.err +} + +func (self *SessionStore) dispatchRequests() { + defer func() { self.done <- true }() + for { + select { + case <-self.quit: + return + case req := <-self.newChan: + req.response <- self.new(req.ctx) + case req := <-self.getChan: + req.response <- self.get(req.user, req.id) + case req := <-self.removeChan: + req.response <- self.remove(req.user, req.id) + } + } +} + +func (self *SessionStore) GetInterface() *SessionStoreChan { + ch := &SessionStoreChan{} + ch.newChan = self.newChan + ch.getChan = self.getChan + ch.removeChan = self.removeChan + return ch +} + +func (self *SessionStore) Cleanup() { + self.quit <- true + <-self.done + close(self.quit) + close(self.done) + close(self.newChan) + close(self.getChan) + close(self.removeChan) +} + +func NewSessionStore(conf *Config) (store *SessionStore, err error) { + store = new(SessionStore) + + store.quit = make(chan bool) + store.done = make(chan bool) + store.store = make(map[string]map[string]*Session) + store.newChan = make(chan newSessionRequest) + store.getChan = make(chan getSessionRequest) + store.removeChan = make(chan removeSessionRequest) + + go store.dispatchRequests() + return +} -- cgit v0.10.2 From 7ba1348b1a9cbfc53f902ea43282d6207fc40313 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 21 Dec 2015 08:45:13 +0100 Subject: minor cleanup for session testing diff --git a/session.go b/session.go index ce84053..8f20958 100644 --- a/session.go +++ b/session.go @@ -149,7 +149,9 @@ func (self *Session) dispatchRequests() { for { select { case <-self.runChan: - self.run() + if !self.running { + self.run() + } case <-self.cancelChan: if self.running { rhdl.Println("Session: canceling running imports is not yet implemented") @@ -168,8 +170,6 @@ func (self *Session) dispatchRequests() { self.running = false rhdl.Printf("Session: import is done: %+v", result) // TODO: call all subscribed done handler - // TODO: send remove request to session store? - return } } } @@ -185,6 +185,7 @@ func (self *Session) getInterface() *SessionChan { func (self *Session) Cleanup() { // TODO: this blocks if dispatchRequests has ended already... + // or if cancel doesn't work... self.cancelChan <- true <-self.done close(self.done) -- cgit v0.10.2 From 0c5eb4132e9c5af37cf073c957e21529487a78f4 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 21 Dec 2015 08:46:24 +0100 Subject: added a cancel channel to fetcher/importer (untestedgit show) diff --git a/fetcher.go b/fetcher.go index a8ac22c..bff62bc 100644 --- a/fetcher.go +++ b/fetcher.go @@ -113,6 +113,10 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro easy.Setopt(curl.OPT_NOPROGRESS, false) easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { + if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + return false + } + ctx := userdata.(*ImportContext) if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(1, "downloading", dlnow/dltotal, ctx.ProgressCallBackData) diff --git a/importer.go b/importer.go index 292d376..9be1257 100644 --- a/importer.go +++ b/importer.go @@ -63,6 +63,7 @@ type ImportContext struct { DeleteSourceDir bool ProgressCallBack ImportProgressCB ProgressCallBackData interface{} + Cancel <-chan bool } func NewImportContext(conf *Config, rddb *RdDbChan, user string) *ImportContext { @@ -86,6 +87,7 @@ func NewImportContext(conf *Config, rddb *RdDbChan, user string) *ImportContext ctx.DeleteSourceFile = false ctx.DeleteSourceDir = false ctx.ProgressCallBack = nil + ctx.Cancel = nil return ctx } @@ -482,6 +484,10 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { easy.Setopt(curl.OPT_NOPROGRESS, false) easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { + if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + return false + } + ctx := userdata.(*ImportContext) if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(2, "importing", ulnow/ultotal, ctx.ProgressCallBackData) -- cgit v0.10.2 From a51ab35bc71f678fcc0bc07bd2d2fd682587381e Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 21 Dec 2015 09:50:28 +0100 Subject: improved sanity checks for importcontext and telnet interface diff --git a/importer.go b/importer.go index 9be1257..df491d7 100644 --- a/importer.go +++ b/importer.go @@ -35,6 +35,11 @@ import ( "path" ) +const ( + CART_MAX = 999999 + CUT_MAX = 999 +) + var ( bool2str = map[bool]string{false: "0", true: "1"} ) @@ -100,6 +105,12 @@ func (ctx *ImportContext) SanityCheck() error { return fmt.Errorf("empty Password on untrusted control interface is not allowed") } if ctx.ShowId != 0 { + if ctx.ShowId != 0 && ctx.ShowId > CART_MAX { + return fmt.Errorf("ShowId %d is outside of allowed range (0 < show-id < %d)", ctx.ShowId, CART_MAX) + } + if ctx.Cart != 0 && ctx.Cart > CART_MAX { + return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) + } return nil } if ctx.GroupName != "" { @@ -118,6 +129,12 @@ func (ctx *ImportContext) SanityCheck() error { if ctx.Cart == 0 { return fmt.Errorf("either ShowId, PoolName or CartNumber must be supplied") } + if ctx.Cart > CART_MAX { + return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) + } + if ctx.Cut != 0 && ctx.Cut > CUT_MAX { + return fmt.Errorf("Cut %d is outside of allowed range (0 < cart < %d)", ctx.Cut, CUT_MAX) + } if ctx.Channels != 1 && ctx.Channels != 2 { return fmt.Errorf("channles must be 1 or 2") } -- cgit v0.10.2 From fa4e8087ca710222069e0353bd591056f03f98b2 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 21 Dec 2015 22:19:27 +0100 Subject: canceling using works now diff --git a/fetcher.go b/fetcher.go index bff62bc..24635d1 100644 --- a/fetcher.go +++ b/fetcher.go @@ -34,7 +34,9 @@ import ( "os" "path" "path/filepath" + "strconv" "strings" + "time" ) type FetchResult struct { @@ -114,10 +116,11 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro easy.Setopt(curl.OPT_NOPROGRESS, false) easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + res.ResponseCode = http.StatusNoContent + res.ErrorString = "canceled" return false } - ctx := userdata.(*ImportContext) if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(1, "downloading", dlnow/dltotal, ctx.ProgressCallBackData) } @@ -126,7 +129,11 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { - err = fmt.Errorf("curl-fetcher('%s'): %s", ctx.SourceUri, err) + if res.ResponseCode == http.StatusNoContent { + err = nil + } else { + err = fmt.Errorf("curl-fetcher('%s'): %s", ctx.SourceUri, err) + } return } @@ -159,6 +166,35 @@ func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err err return } +func FetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { + rhdl.Printf("Fake fetcher for '%s'", ctx.SourceUri) + + if duration, err := strconv.ParseUint(uri.Host, 10, 32); err != nil { + err = nil + res.ResponseCode = http.StatusBadRequest + res.ErrorString = "invalid duration (must be a positive integer)" + } else { + for i := uint(0); i < uint(duration); i++ { + if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + res.ResponseCode = http.StatusNoContent + res.ErrorString = "canceled" + return nil + } + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(1, "faking", float64(i)/float64(duration), ctx.ProgressCallBackData) + } + time.Sleep(100 * time.Millisecond) + } + if ctx.ProgressCallBack != nil { + ctx.ProgressCallBack(42, "faking", 1.0, ctx.ProgressCallBackData) + } + ctx.SourceFile = "/nonexistend/fake.mp3" + ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false + } + return nil +} + type FetchFunc func(*ImportContext, *FetchResult, *url.URL) (err error) // TODO: implement fetchers for: @@ -168,6 +204,7 @@ type FetchFunc func(*ImportContext, *FetchResult, *url.URL) (err error) var ( fetchers = map[string]FetchFunc{ "local": FetchFileLocal, + "fake": FetchFileFake, } curl_protos = map[string]bool{ "http": false, "https": false, diff --git a/importer.go b/importer.go index df491d7..512daf6 100644 --- a/importer.go +++ b/importer.go @@ -502,10 +502,11 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { easy.Setopt(curl.OPT_NOPROGRESS, false) easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + res.ResponseCode = http.StatusNoContent + res.ErrorString = "canceled" return false } - ctx := userdata.(*ImportContext) if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(2, "importing", ulnow/ultotal, ctx.ProgressCallBackData) } @@ -514,7 +515,13 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { - err = fmt.Errorf("importer: %s", err) + if res.ResponseCode == http.StatusNoContent { + res.Cart = ctx.Cart + res.Cut = ctx.Cut + err = nil + } else { + err = fmt.Errorf("importer: %s", err) + } return } -- cgit v0.10.2 From 4a358bab91c53ac951567c0f25d08368095b843a Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 21 Dec 2015 22:41:22 +0100 Subject: improved error handling and reporting, canceling running requests when telnet client leaves diff --git a/fetcher.go b/fetcher.go index 24635d1..1f0b364 100644 --- a/fetcher.go +++ b/fetcher.go @@ -116,6 +116,7 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro easy.Setopt(curl.OPT_NOPROGRESS, false) easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + rhl.Printf("downloading '%s' got canceled", ctx.SourceUri) res.ResponseCode = http.StatusNoContent res.ErrorString = "canceled" return false @@ -129,6 +130,11 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro easy.Setopt(curl.OPT_PROGRESSDATA, ctx) if err = easy.Perform(); err != nil { + if cbdata.File != nil { + rhdl.Printf("Removing stale file: %s", cbdata.filename) + os.Remove(cbdata.filename) + os.Remove(path.Dir(cbdata.filename)) + } if res.ResponseCode == http.StatusNoContent { err = nil } else { @@ -176,6 +182,7 @@ func FetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { } else { for i := uint(0); i < uint(duration); i++ { if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + rhl.Printf("faking got canceled") res.ResponseCode = http.StatusNoContent res.ErrorString = "canceled" return nil diff --git a/importer.go b/importer.go index 512daf6..1bfdf0c 100644 --- a/importer.go +++ b/importer.go @@ -516,6 +516,7 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { if err = easy.Perform(); err != nil { if res.ResponseCode == http.StatusNoContent { + rhl.Printf("import to cart/cat %d/%d got canceled", ctx.Cart, ctx.Cut) res.Cart = ctx.Cart res.Cut = ctx.Cut err = nil @@ -626,7 +627,7 @@ func cleanup_files(ctx *ImportContext) { func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { defer cleanup_files(ctx) - rhdl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) + rhl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) if ctx.ProgressCallBack != nil { ctx.ProgressCallBack(2, "importing", 0.0, ctx.ProgressCallBackData) @@ -696,6 +697,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { return } if res.ResponseCode != http.StatusOK { + rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) rmres := ImportResult{ResponseCode: http.StatusOK} if rmCartOnErr { if err = remove_cart(ctx, &rmres); err != nil { @@ -706,10 +708,12 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { return } } + } else { + rhl.Printf("File got succesfully imported into Cart/Cut %d/%d", res.Cart, res.Cut) } } else { res.ResponseCode = http.StatusBadRequest - res.ErrorString = "The request doesn't contain enough information to be processed" + res.ErrorString = "importer: The request doesn't contain enough information to be processed" } return -- cgit v0.10.2 From bcc3483ff7e719c4e429b22e045937fd19e34400 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 22 Dec 2015 00:45:49 +0100 Subject: basic session handling works now diff --git a/fetcher.go b/fetcher.go index 1f0b364..36d7055 100644 --- a/fetcher.go +++ b/fetcher.go @@ -193,7 +193,7 @@ func FetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { time.Sleep(100 * time.Millisecond) } if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(42, "faking", 1.0, ctx.ProgressCallBackData) + ctx.ProgressCallBack(1, "faking", 1.0, ctx.ProgressCallBackData) } ctx.SourceFile = "/nonexistend/fake.mp3" ctx.DeleteSourceFile = false diff --git a/importer.go b/importer.go index 1bfdf0c..a245270 100644 --- a/importer.go +++ b/importer.go @@ -693,18 +693,20 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } if ctx.Cart != 0 && ctx.Cut != 0 { - if err = import_audio(ctx, res); err != nil { - return - } - if res.ResponseCode != http.StatusOK { - rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) + if err = import_audio(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err != nil { + rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", ctx.Cart, ctx.Cut, err) + } else { + rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) + } + // Try to clean up after failed import rmres := ImportResult{ResponseCode: http.StatusOK} if rmCartOnErr { - if err = remove_cart(ctx, &rmres); err != nil { + if rerr := remove_cart(ctx, &rmres); rerr != nil { return } } else if rmCutOnErr { - if err = remove_cut(ctx, &rmres); err != nil { + if rerr := remove_cut(ctx, &rmres); rerr != nil { return } } diff --git a/session.go b/session.go index 8f20958..08a33ce 100644 --- a/session.go +++ b/session.go @@ -25,7 +25,14 @@ package rhimport import ( - "time" + "net/http" +) + +const ( + SESSION_NEW = iota + SESSION_RUNNING + SESSION_CANCELED + SESSION_DONE ) type SessionChan struct { @@ -37,8 +44,9 @@ type SessionChan struct { type Session struct { ctx ImportContext - running bool + state int done chan bool + cancelIntChan chan bool progressIntChan chan ProgressData doneIntChan chan ImportResult runChan chan bool @@ -79,26 +87,35 @@ func session_progress_callback(step int, step_name string, progress float64, use out <- ProgressData{step, step_name, progress} } -// TODO: actually call import here func session_import_run(ctx ImportContext, done chan<- ImportResult) { - rhdl.Printf("faking import for: %+v", ctx) - for i := 0; i < 100; i++ { - if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(42, "faking", float64(i)/100.0, ctx.ProgressCallBackData) - } - time.Sleep(100 * time.Millisecond) + if err := ctx.SanityCheck(); err != nil { + done <- ImportResult{http.StatusBadRequest, err.Error(), 0, 0} + return } - if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(42, "faking", 1.0, ctx.ProgressCallBackData) + + if res, err := FetchFile(&ctx); err != nil { + done <- ImportResult{http.StatusInternalServerError, err.Error(), 0, 0} + return + } else if res.ResponseCode != http.StatusOK { + done <- ImportResult{res.ResponseCode, res.ErrorString, 0, 0} + return + } + + if res, err := ImportFile(&ctx); err != nil { + done <- ImportResult{http.StatusInternalServerError, err.Error(), 0, 0} + return + } else { + done <- *res + return } - done <- ImportResult{200, "OK", 0, 0} } func (self *Session) run() { self.ctx.ProgressCallBack = session_progress_callback self.ctx.ProgressCallBackData = (chan<- ProgressData)(self.progressIntChan) + self.ctx.Cancel = self.cancelIntChan go session_import_run(self.ctx, self.doneIntChan) - self.running = true + self.state = SESSION_RUNNING return } @@ -149,13 +166,16 @@ func (self *Session) dispatchRequests() { for { select { case <-self.runChan: - if !self.running { + if self.state == SESSION_NEW { self.run() } case <-self.cancelChan: - if self.running { - rhdl.Println("Session: canceling running imports is not yet implemented") - // TODO: send cancel to import goroutine once this got implemented + if self.state == SESSION_RUNNING { + rhdl.Println("Session: canceling running import") + select { + case self.cancelIntChan <- true: + default: // session got canceled already + } } else { return } @@ -167,7 +187,7 @@ func (self *Session) dispatchRequests() { rhdl.Printf("Session: got progress: %+v", progress) // TODO: call all subscribed progress handler case result := <-self.doneIntChan: - self.running = false + self.state = SESSION_DONE rhdl.Printf("Session: import is done: %+v", result) // TODO: call all subscribed done handler } @@ -184,9 +204,8 @@ func (self *Session) getInterface() *SessionChan { } func (self *Session) Cleanup() { - // TODO: this blocks if dispatchRequests has ended already... - // or if cancel doesn't work... self.cancelChan <- true + rhdl.Printf("waiting for session to close") <-self.done close(self.done) close(self.progressIntChan) @@ -195,13 +214,15 @@ func (self *Session) Cleanup() { close(self.cancelChan) close(self.addProgressChan) close(self.addDoneChan) + rhdl.Printf("session is now cleaned up") } func NewSession(ctx *ImportContext) (session *Session) { session = new(Session) - session.running = false + session.state = SESSION_NEW session.ctx = *ctx session.done = make(chan bool) + session.cancelIntChan = make(chan bool, 1) session.progressIntChan = make(chan ProgressData) session.doneIntChan = make(chan ImportResult) session.runChan = make(chan bool) diff --git a/session_store.go b/session_store.go index b2e1538..8dd74c7 100644 --- a/session_store.go +++ b/session_store.go @@ -130,7 +130,7 @@ func (self *SessionStoreChan) Get(user, id string) (*SessionChan, error) { func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { if session, exists := self.store[user][id]; exists { - session.Cleanup() + go session.Cleanup() // cleanup could take a while -> don't block all the other stuff delete(self.store[user], id) rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) if userstore, exists := self.store[user]; exists { -- cgit v0.10.2 From 5363f23b19df89eaf7fc8d0c250ab9db9b2ab8b3 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 22 Dec 2015 04:18:46 +0100 Subject: session based callbacks work now diff --git a/fetcher.go b/fetcher.go index 36d7055..acb6592 100644 --- a/fetcher.go +++ b/fetcher.go @@ -123,7 +123,9 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro } if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(1, "downloading", dlnow/dltotal, ctx.ProgressCallBackData) + if keep := ctx.ProgressCallBack(1, "downloading", dlnow/dltotal, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } } return true }) @@ -156,7 +158,9 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData) + if keep := ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } } ctx.SourceFile = filepath.Join(ctx.conf.LocalFetchDir, path.Clean("/"+uri.Path)) @@ -188,12 +192,16 @@ func FetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { return nil } if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(1, "faking", float64(i)/float64(duration), ctx.ProgressCallBackData) + if keep := ctx.ProgressCallBack(1, "faking", float64(i)/float64(duration), ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } } time.Sleep(100 * time.Millisecond) } if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(1, "faking", 1.0, ctx.ProgressCallBackData) + if keep := ctx.ProgressCallBack(1, "faking", 1.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } } ctx.SourceFile = "/nonexistend/fake.mp3" ctx.DeleteSourceFile = false diff --git a/importer.go b/importer.go index a245270..575adc9 100644 --- a/importer.go +++ b/importer.go @@ -44,7 +44,7 @@ var ( bool2str = map[bool]string{false: "0", true: "1"} ) -type ImportProgressCB func(step int, step_name string, progress float64, userdata interface{}) +type ImportProgressCB func(step int, step_name string, progress float64, userdata interface{}) bool type ImportContext struct { conf *Config @@ -508,7 +508,9 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { } if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(2, "importing", ulnow/ultotal, ctx.ProgressCallBackData) + if keep := ctx.ProgressCallBack(2, "importing", ulnow/ultotal, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } } return true }) @@ -630,7 +632,9 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { rhl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) if ctx.ProgressCallBack != nil { - ctx.ProgressCallBack(2, "importing", 0.0, ctx.ProgressCallBackData) + if keep := ctx.ProgressCallBack(2, "importing", 0.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } } if ctx.Trusted { diff --git a/session.go b/session.go index 08a33ce..864aac6 100644 --- a/session.go +++ b/session.go @@ -53,7 +53,18 @@ type Session struct { cancelChan chan bool addProgressChan chan sessionAddProgressHandlerRequest addDoneChan chan sessionAddDoneHandlerRequest - // TODO: add pub/sub for progress and done + progressCBs []*SessionProgressCB + doneCBs []*SessionDoneCB +} + +type SessionProgressCB struct { + cb ImportProgressCB + userdata interface{} +} + +type SessionDoneCB struct { + cb func(ImportResult, interface{}) bool + userdata interface{} } type ProgressData struct { @@ -78,13 +89,14 @@ type sessionAddDoneHandlerResponse struct { type sessionAddDoneHandlerRequest struct { userdata interface{} - callback func(*ImportResult, interface{}) + callback func(ImportResult, interface{}) bool response chan<- sessionAddDoneHandlerResponse } -func session_progress_callback(step int, step_name string, progress float64, userdata interface{}) { +func session_progress_callback(step int, step_name string, progress float64, userdata interface{}) bool { out := userdata.(chan<- ProgressData) out <- ProgressData{step, step_name, progress} + return true } func session_import_run(ctx ImportContext, done chan<- ImportResult) { @@ -129,6 +141,7 @@ func (self *SessionChan) Cancel() { func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressCB) (resp sessionAddProgressHandlerResponse) { rhdl.Printf("Session: addProgressHandler called with: %+v", userdata) + self.progressCBs = append(self.progressCBs, &SessionProgressCB{cb, userdata}) return } @@ -144,12 +157,13 @@ func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgr return res.err } -func (self *Session) addDoneHandler(userdata interface{}, cb func(*ImportResult, interface{})) (resp sessionAddDoneHandlerResponse) { +func (self *Session) addDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) (resp sessionAddDoneHandlerResponse) { rhdl.Printf("Session: addDoneHandler called with: %+v", userdata) + self.doneCBs = append(self.doneCBs, &SessionDoneCB{cb, userdata}) return } -func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(*ImportResult, interface{})) error { +func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) error { res_ch := make(chan sessionAddDoneHandlerResponse) req := sessionAddDoneHandlerRequest{} req.userdata = userdata @@ -183,13 +197,23 @@ func (self *Session) dispatchRequests() { req.response <- self.addProgressHandler(req.userdata, req.callback) case req := <-self.addDoneChan: req.response <- self.addDoneHandler(req.userdata, req.callback) - case progress := <-self.progressIntChan: - rhdl.Printf("Session: got progress: %+v", progress) - // TODO: call all subscribed progress handler - case result := <-self.doneIntChan: + case p := <-self.progressIntChan: + for _, cb := range self.progressCBs { + if cb.cb != nil { + if keep := cb.cb(p.step, p.step_name, p.progress, cb.userdata); !keep { + cb.cb = nil + } + } + } + case r := <-self.doneIntChan: self.state = SESSION_DONE - rhdl.Printf("Session: import is done: %+v", result) - // TODO: call all subscribed done handler + for _, cb := range self.doneCBs { + if cb.cb != nil { + if keep := cb.cb(r, cb.userdata); !keep { + cb.cb = nil + } + } + } } } } -- cgit v0.10.2 From ba5e2bbc79d613d995ad508d9dab818d4ab8112e Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 22 Dec 2015 04:21:32 +0100 Subject: added TODO diff --git a/importer.go b/importer.go index 575adc9..3ee94cd 100644 --- a/importer.go +++ b/importer.go @@ -637,6 +637,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } } + // TODO: on trusted interfaces we should call getPassword again with cached=false after 401's if ctx.Trusted { if err = ctx.getPassword(true); err != nil { return -- cgit v0.10.2 From 5ef7fd6e20f360ff370a5e7e082b5d004c599fdc Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 22 Dec 2015 23:01:47 +0100 Subject: session is now safe to be used even if it is about to be removed diff --git a/session.go b/session.go index 864aac6..8843c81 100644 --- a/session.go +++ b/session.go @@ -25,6 +25,7 @@ package rhimport import ( + "fmt" "net/http" ) @@ -132,15 +133,23 @@ func (self *Session) run() { } func (self *SessionChan) Run() { - self.runChan <- true + select { + case self.runChan <- true: + default: // command is already pending or session is about to be closed/removed + } } func (self *SessionChan) Cancel() { - self.cancelChan <- true + select { + case self.cancelChan <- true: + default: // cancel is already pending or session is about to be closed/removed + } } func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressCB) (resp sessionAddProgressHandlerResponse) { - rhdl.Printf("Session: addProgressHandler called with: %+v", userdata) + if self.state != SESSION_NEW && self.state != SESSION_RUNNING { + resp.err = fmt.Errorf("session is already done/canceled") + } self.progressCBs = append(self.progressCBs, &SessionProgressCB{cb, userdata}) return } @@ -151,14 +160,20 @@ func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgr req.userdata = userdata req.callback = cb req.response = res_ch - self.addProgressChan <- req + select { + case self.addProgressChan <- req: + default: + return fmt.Errorf("session is about to be closed/removed") + } res := <-res_ch return res.err } func (self *Session) addDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) (resp sessionAddDoneHandlerResponse) { - rhdl.Printf("Session: addDoneHandler called with: %+v", userdata) + if self.state == SESSION_NEW && self.state != SESSION_RUNNING { + resp.err = fmt.Errorf("session is already done/canceled") + } self.doneCBs = append(self.doneCBs, &SessionDoneCB{cb, userdata}) return } @@ -169,12 +184,36 @@ func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResu req.userdata = userdata req.callback = cb req.response = res_ch - self.addDoneChan <- req + select { + case self.addDoneChan <- req: + default: + return fmt.Errorf("session is about to be closed/removed") + } res := <-res_ch return res.err } +func (self *Session) callProgressHandler(p *ProgressData) { + for _, cb := range self.progressCBs { + if cb.cb != nil { + if keep := cb.cb(p.step, p.step_name, p.progress, cb.userdata); !keep { + cb.cb = nil + } + } + } +} + +func (self *Session) callDoneHandler(r *ImportResult) { + for _, cb := range self.doneCBs { + if cb.cb != nil { + if keep := cb.cb(*r, cb.userdata); !keep { + cb.cb = nil + } + } + } +} + func (self *Session) dispatchRequests() { defer func() { self.done <- true }() for { @@ -188,8 +227,9 @@ func (self *Session) dispatchRequests() { rhdl.Println("Session: canceling running import") select { case self.cancelIntChan <- true: - default: // session got canceled already + default: // session got canceled already?? } + self.state = SESSION_CANCELED } else { return } @@ -198,22 +238,10 @@ func (self *Session) dispatchRequests() { case req := <-self.addDoneChan: req.response <- self.addDoneHandler(req.userdata, req.callback) case p := <-self.progressIntChan: - for _, cb := range self.progressCBs { - if cb.cb != nil { - if keep := cb.cb(p.step, p.step_name, p.progress, cb.userdata); !keep { - cb.cb = nil - } - } - } + self.callProgressHandler(&p) case r := <-self.doneIntChan: self.state = SESSION_DONE - for _, cb := range self.doneCBs { - if cb.cb != nil { - if keep := cb.cb(r, cb.userdata); !keep { - cb.cb = nil - } - } - } + self.callDoneHandler(&r) } } } @@ -234,10 +262,12 @@ func (self *Session) Cleanup() { close(self.done) close(self.progressIntChan) close(self.doneIntChan) - close(self.runChan) - close(self.cancelChan) - close(self.addProgressChan) - close(self.addDoneChan) + // don't close the channels we give out because this might lead to a panic if + // somebody wites to an already removed session + // close(self.runChan) + // close(self.cancelChan) + // close(self.addProgressChan) + // close(self.addDoneChan) rhdl.Printf("session is now cleaned up") } @@ -247,12 +277,12 @@ func NewSession(ctx *ImportContext) (session *Session) { session.ctx = *ctx session.done = make(chan bool) session.cancelIntChan = make(chan bool, 1) - session.progressIntChan = make(chan ProgressData) - session.doneIntChan = make(chan ImportResult) - session.runChan = make(chan bool) - session.cancelChan = make(chan bool) - session.addProgressChan = make(chan sessionAddProgressHandlerRequest) - session.addDoneChan = make(chan sessionAddDoneHandlerRequest) + session.progressIntChan = make(chan ProgressData, 10) + session.doneIntChan = make(chan ImportResult, 1) + session.runChan = make(chan bool, 1) + session.cancelChan = make(chan bool, 1) + session.addProgressChan = make(chan sessionAddProgressHandlerRequest, 10) + session.addDoneChan = make(chan sessionAddDoneHandlerRequest, 10) go session.dispatchRequests() return } -- cgit v0.10.2 From 87767fb705b253dffb2d88b44ae082302935bd0e Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 23 Dec 2015 00:00:55 +0100 Subject: sessions now call remove when done diff --git a/session.go b/session.go index 8843c81..bce1072 100644 --- a/session.go +++ b/session.go @@ -46,7 +46,9 @@ type SessionChan struct { type Session struct { ctx ImportContext state int + removeFunc func() done chan bool + quit chan bool cancelIntChan chan bool progressIntChan chan ProgressData doneIntChan chan ImportResult @@ -139,6 +141,15 @@ func (self *SessionChan) Run() { } } +func (self *Session) cancel() { + rhdl.Println("Session: canceling running import") + select { + case self.cancelIntChan <- true: + default: // session got canceled already?? + } + self.state = SESSION_CANCELED +} + func (self *SessionChan) Cancel() { select { case self.cancelChan <- true: @@ -178,6 +189,16 @@ func (self *Session) addDoneHandler(userdata interface{}, cb func(ImportResult, return } +func (self *Session) callProgressHandler(p *ProgressData) { + for _, cb := range self.progressCBs { + if cb.cb != nil { + if keep := cb.cb(p.step, p.step_name, p.progress, cb.userdata); !keep { + cb.cb = nil + } + } + } +} + func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) error { res_ch := make(chan sessionAddDoneHandlerResponse) req := sessionAddDoneHandlerRequest{} @@ -194,16 +215,6 @@ func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResu return res.err } -func (self *Session) callProgressHandler(p *ProgressData) { - for _, cb := range self.progressCBs { - if cb.cb != nil { - if keep := cb.cb(p.step, p.step_name, p.progress, cb.userdata); !keep { - cb.cb = nil - } - } - } -} - func (self *Session) callDoneHandler(r *ImportResult) { for _, cb := range self.doneCBs { if cb.cb != nil { @@ -218,20 +229,18 @@ func (self *Session) dispatchRequests() { defer func() { self.done <- true }() for { select { + case <-self.quit: + if self.state == SESSION_RUNNING { + self.cancel() + } + return case <-self.runChan: if self.state == SESSION_NEW { self.run() } case <-self.cancelChan: if self.state == SESSION_RUNNING { - rhdl.Println("Session: canceling running import") - select { - case self.cancelIntChan <- true: - default: // session got canceled already?? - } - self.state = SESSION_CANCELED - } else { - return + self.cancel() } case req := <-self.addProgressChan: req.response <- self.addProgressHandler(req.userdata, req.callback) @@ -242,6 +251,9 @@ func (self *Session) dispatchRequests() { case r := <-self.doneIntChan: self.state = SESSION_DONE self.callDoneHandler(&r) + if self.removeFunc != nil { + self.removeFunc() + } } } } @@ -256,7 +268,7 @@ func (self *Session) getInterface() *SessionChan { } func (self *Session) Cleanup() { - self.cancelChan <- true + self.quit <- true rhdl.Printf("waiting for session to close") <-self.done close(self.done) @@ -271,9 +283,10 @@ func (self *Session) Cleanup() { rhdl.Printf("session is now cleaned up") } -func NewSession(ctx *ImportContext) (session *Session) { +func NewSession(ctx *ImportContext, removeFunc func()) (session *Session) { session = new(Session) session.state = SESSION_NEW + session.removeFunc = removeFunc session.ctx = *ctx session.done = make(chan bool) session.cancelIntChan = make(chan bool, 1) diff --git a/session_store.go b/session_store.go index 8dd74c7..505bd75 100644 --- a/session_store.go +++ b/session_store.go @@ -84,7 +84,7 @@ func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { if _, exists := self.store[ctx.UserName]; !exists { self.store[ctx.UserName] = make(map[string]*Session) } - self.store[ctx.UserName][resp.id] = NewSession(ctx) + self.store[ctx.UserName][resp.id] = NewSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }) resp.session = self.store[ctx.UserName][resp.id].getInterface() rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) return @@ -197,9 +197,9 @@ func NewSessionStore(conf *Config) (store *SessionStore, err error) { store.quit = make(chan bool) store.done = make(chan bool) store.store = make(map[string]map[string]*Session) - store.newChan = make(chan newSessionRequest) - store.getChan = make(chan getSessionRequest) - store.removeChan = make(chan removeSessionRequest) + store.newChan = make(chan newSessionRequest, 10) + store.getChan = make(chan getSessionRequest, 10) + store.removeChan = make(chan removeSessionRequest, 10) go store.dispatchRequests() return -- cgit v0.10.2 From d78058f0a80c4dc8e5b5f8a1300d7960bf93a078 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 23 Dec 2015 00:01:20 +0100 Subject: rddb channels are now buffered diff --git a/rddb.go b/rddb.go index bb481f4..514abdf 100644 --- a/rddb.go +++ b/rddb.go @@ -339,11 +339,11 @@ func NewRdDb(conf *Config) (db *RdDb, err error) { db.quit = make(chan bool) db.done = make(chan bool) db.password_cache = make(map[string]string) - db.getPasswordChan = make(chan getPasswordRequest) - db.getGroupOfCartChan = make(chan getGroupOfCartRequest) - db.getShowInfoChan = make(chan getShowInfoRequest) - db.checkMusicGroupChan = make(chan checkMusicGroupRequest) - db.getMusicInfoChan = make(chan getMusicInfoRequest) + db.getPasswordChan = make(chan getPasswordRequest, 10) + db.getGroupOfCartChan = make(chan getGroupOfCartRequest, 10) + db.getShowInfoChan = make(chan getShowInfoRequest, 10) + db.checkMusicGroupChan = make(chan checkMusicGroupRequest, 10) + db.getMusicInfoChan = make(chan getMusicInfoRequest, 10) if err = db.init(conf); err != nil { return -- cgit v0.10.2 From 3822025a7a4103f2c2de70f0c9199bc9a64cc3b4 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 23 Dec 2015 01:30:03 +0100 Subject: session now has a timeout diff --git a/session.go b/session.go index bce1072..88648f7 100644 --- a/session.go +++ b/session.go @@ -27,6 +27,7 @@ package rhimport import ( "fmt" "net/http" + "time" ) const ( @@ -34,10 +35,11 @@ const ( SESSION_RUNNING SESSION_CANCELED SESSION_DONE + SESSION_TIMEOUT ) type SessionChan struct { - runChan chan<- bool + runChan chan<- time.Duration cancelChan chan<- bool addProgressChan chan<- sessionAddProgressHandlerRequest addDoneChan chan<- sessionAddDoneHandlerRequest @@ -49,10 +51,11 @@ type Session struct { removeFunc func() done chan bool quit chan bool + timer *time.Timer cancelIntChan chan bool progressIntChan chan ProgressData doneIntChan chan ImportResult - runChan chan bool + runChan chan time.Duration cancelChan chan bool addProgressChan chan sessionAddProgressHandlerRequest addDoneChan chan sessionAddDoneHandlerRequest @@ -125,18 +128,19 @@ func session_import_run(ctx ImportContext, done chan<- ImportResult) { } } -func (self *Session) run() { +func (self *Session) run(timeout time.Duration) { self.ctx.ProgressCallBack = session_progress_callback self.ctx.ProgressCallBackData = (chan<- ProgressData)(self.progressIntChan) self.ctx.Cancel = self.cancelIntChan go session_import_run(self.ctx, self.doneIntChan) self.state = SESSION_RUNNING + self.timer.Reset(timeout) return } -func (self *SessionChan) Run() { +func (self *SessionChan) Run(timeout time.Duration) { select { - case self.runChan <- true: + case self.runChan <- timeout: default: // command is already pending or session is about to be closed/removed } } @@ -182,7 +186,7 @@ func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgr } func (self *Session) addDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) (resp sessionAddDoneHandlerResponse) { - if self.state == SESSION_NEW && self.state != SESSION_RUNNING { + if self.state != SESSION_NEW && self.state != SESSION_RUNNING { resp.err = fmt.Errorf("session is already done/canceled") } self.doneCBs = append(self.doneCBs, &SessionDoneCB{cb, userdata}) @@ -234,9 +238,19 @@ func (self *Session) dispatchRequests() { self.cancel() } return - case <-self.runChan: + case <-self.timer.C: + if self.state == SESSION_RUNNING { + self.cancel() + } + self.state = SESSION_TIMEOUT + r := &ImportResult{500, "timeout", 0, 0} + self.callDoneHandler(r) + if self.removeFunc != nil { + self.removeFunc() + } + case t := <-self.runChan: if self.state == SESSION_NEW { - self.run() + self.run(t) } case <-self.cancelChan: if self.state == SESSION_RUNNING { @@ -249,10 +263,13 @@ func (self *Session) dispatchRequests() { case p := <-self.progressIntChan: self.callProgressHandler(&p) case r := <-self.doneIntChan: - self.state = SESSION_DONE - self.callDoneHandler(&r) - if self.removeFunc != nil { - self.removeFunc() + if self.state != SESSION_TIMEOUT { + self.timer.Stop() + self.state = SESSION_DONE + self.callDoneHandler(&r) + if self.removeFunc != nil { + self.removeFunc() + } } } } @@ -271,11 +288,14 @@ func (self *Session) Cleanup() { self.quit <- true rhdl.Printf("waiting for session to close") <-self.done + close(self.quit) close(self.done) - close(self.progressIntChan) - close(self.doneIntChan) + self.timer.Stop() // don't close the channels we give out because this might lead to a panic if // somebody wites to an already removed session + // close(self.cancelIntChan) + // close(self.progressIntChan) + // close(self.doneIntChan) // close(self.runChan) // close(self.cancelChan) // close(self.addProgressChan) @@ -289,10 +309,11 @@ func NewSession(ctx *ImportContext, removeFunc func()) (session *Session) { session.removeFunc = removeFunc session.ctx = *ctx session.done = make(chan bool) + session.timer = time.NewTimer(10 * time.Second) session.cancelIntChan = make(chan bool, 1) session.progressIntChan = make(chan ProgressData, 10) session.doneIntChan = make(chan ImportResult, 1) - session.runChan = make(chan bool, 1) + session.runChan = make(chan time.Duration, 1) session.cancelChan = make(chan bool, 1) session.addProgressChan = make(chan sessionAddProgressHandlerRequest, 10) session.addDoneChan = make(chan sessionAddDoneHandlerRequest, 10) -- cgit v0.10.2 From 7f63f817486684c0dcb440bf57ff79c07c487911 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 25 Dec 2015 22:13:49 +0100 Subject: telnet control interface now uses spreadspace/telgo diff --git a/importer.go b/importer.go index 3ee94cd..7ce4033 100644 --- a/importer.go +++ b/importer.go @@ -119,7 +119,7 @@ func (ctx *ImportContext) SanityCheck() error { return err } if !ismusic { - return fmt.Errorf("supplied GroupName is not a music pool") + return fmt.Errorf("supplied GroupName '%s' is not a music pool", ctx.GroupName) } if ctx.Cart != 0 || ctx.Cut != 0 { return fmt.Errorf("Cart and Cut must not be supplied when importing into a music group") diff --git a/session_store.go b/session_store.go index 505bd75..2aabc44 100644 --- a/session_store.go +++ b/session_store.go @@ -79,6 +79,7 @@ type SessionStore struct { } func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { + // TODO: for untrusted interfaces we need to check Username and PassWord!!!! b := uuid.NewV4().Bytes() resp.id = strings.ToLower(strings.TrimRight(base32.StdEncoding.EncodeToString(b), "=")) if _, exists := self.store[ctx.UserName]; !exists { -- cgit v0.10.2 From e2dee26e41817302ed4e14207ce779d58004f797 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 26 Dec 2015 09:58:34 +0100 Subject: session store now checks password diff --git a/fetcher.go b/fetcher.go index acb6592..81072eb 100644 --- a/fetcher.go +++ b/fetcher.go @@ -247,31 +247,14 @@ func fetcher_init() { } func checkPassword(ctx *ImportContext, result *FetchResult) (err error) { - cached := true - - for { - res_ch := make(chan getPasswordResult) - req := getPasswordRequest{} - req.username = ctx.UserName - req.cached = cached - req.response = res_ch - ctx.rddb.getPasswordChan <- req - - res := <-res_ch - if res.err != nil { - return res.err - } - if ctx.Password == res.password { - return nil - } - if cached { - cached = false - } else { - break - } + ok := false + if ok, err = ctx.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { + return + } + if !ok { + result.ResponseCode = http.StatusUnauthorized + result.ErrorString = "invalid username and/or password" } - result.ResponseCode = http.StatusUnauthorized - result.ErrorString = "invalid username and/or password" return } diff --git a/rddb.go b/rddb.go index 514abdf..7466d9c 100644 --- a/rddb.go +++ b/rddb.go @@ -177,6 +177,33 @@ func (self *RdDb) getPassword(username string, cached bool) (result getPasswordR return } +func (self *RdDbChan) CheckPassword(username, password string) (result bool, err error) { + cached := true + + for { + res_ch := make(chan getPasswordResult) + req := getPasswordRequest{} + req.username = username + req.cached = cached + req.response = res_ch + self.getPasswordChan <- req + + res := <-res_ch + if res.err != nil { + return false, res.err + } + if password == res.password { + return true, nil + } + if cached { + cached = false + } else { + break + } + } + return false, nil +} + func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { var rows *sql.Rows if rows, result.err = self.getGroupOfCartStmt.Query(cart, cart); result.err != nil { diff --git a/session_store.go b/session_store.go index 2aabc44..e065182 100644 --- a/session_store.go +++ b/session_store.go @@ -79,7 +79,15 @@ type SessionStore struct { } func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { - // TODO: for untrusted interfaces we need to check Username and PassWord!!!! + if !ctx.Trusted { + if ok, err := ctx.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { + resp.err = err + return + } else if !ok { + resp.err = fmt.Errorf("invalid username and/or password") + return + } + } b := uuid.NewV4().Bytes() resp.id = strings.ToLower(strings.TrimRight(base32.StdEncoding.EncodeToString(b), "=")) if _, exists := self.store[ctx.UserName]; !exists { -- cgit v0.10.2 From fb26578868ae72ac636c8f5b286b968bfce4aa0e Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sat, 26 Dec 2015 10:52:33 +0100 Subject: small refactoring diff --git a/importer.go b/importer.go index 7ce4033..28ca55a 100644 --- a/importer.go +++ b/importer.go @@ -142,84 +142,28 @@ func (ctx *ImportContext) SanityCheck() error { } func (ctx *ImportContext) getPassword(cached bool) (err error) { - res_ch := make(chan getPasswordResult) - req := getPasswordRequest{} - req.username = ctx.UserName - req.cached = cached - req.response = res_ch - ctx.rddb.getPasswordChan <- req - - res := <-res_ch - if res.err != nil { - return res.err - } - ctx.Password = res.password - return nil + ctx.Password, err = ctx.rddb.GetPassword(ctx.UserName, cached) + return } -func (ctx *ImportContext) getGroupOfCart() error { - res_ch := make(chan getGroupOfCartResult) - req := getGroupOfCartRequest{} - req.cart = ctx.Cart - req.response = res_ch - ctx.rddb.getGroupOfCartChan <- req - - res := <-res_ch - if res.err != nil { - return res.err - } - ctx.GroupName = res.group - return nil +func (ctx *ImportContext) getGroupOfCart() (err error) { + ctx.GroupName, err = ctx.rddb.GetGroupOfCart(ctx.Cart) + return } func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { - res_ch := make(chan getShowInfoResult) - req := getShowInfoRequest{} - req.showid = ctx.ShowId - req.response = res_ch - ctx.rddb.getShowInfoChan <- req - - res := <-res_ch - if res.err != nil { - err = res.err - return - } - ctx.GroupName = res.group - ctx.NormalizationLevel = res.norm_lvl - ctx.AutotrimLevel = res.trim_lvl + ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.rddb.GetShowInfo(ctx.ShowId) ctx.Channels = 2 ctx.UseMetaData = true - carts = res.carts return } func (ctx *ImportContext) checkMusicGroup() (bool, error) { - res_ch := make(chan checkMusicGroupResult) - req := checkMusicGroupRequest{} - req.group = ctx.GroupName - req.response = res_ch - ctx.rddb.checkMusicGroupChan <- req - - res := <-res_ch - if res.err != nil { - return false, res.err - } - return res.ismusic, nil + return ctx.rddb.CheckMusicGroup(ctx.GroupName) } func (ctx *ImportContext) getMusicInfo() (err error) { - res_ch := make(chan getMusicInfoResult) - req := getMusicInfoRequest{} - req.group = ctx.GroupName - req.response = res_ch - ctx.rddb.getMusicInfoChan <- req - - res := <-res_ch - if res.err != nil { - return res.err - } - ctx.NormalizationLevel = res.norm_lvl - ctx.AutotrimLevel = res.trim_lvl + ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.rddb.GetMusicInfo(ctx.GroupName) ctx.Channels = 2 ctx.UseMetaData = true ctx.Cart = 0 diff --git a/rddb.go b/rddb.go index 7466d9c..ead1ac0 100644 --- a/rddb.go +++ b/rddb.go @@ -177,7 +177,22 @@ func (self *RdDb) getPassword(username string, cached bool) (result getPasswordR return } -func (self *RdDbChan) CheckPassword(username, password string) (result bool, err error) { +func (self *RdDbChan) GetPassword(username string, cached bool) (string, error) { + res_ch := make(chan getPasswordResult) + req := getPasswordRequest{} + req.username = username + req.cached = cached + req.response = res_ch + self.getPasswordChan <- req + + res := <-res_ch + if res.err != nil { + return "", res.err + } + return res.password, nil +} + +func (self *RdDbChan) CheckPassword(username, password string) (bool, error) { cached := true for { @@ -234,6 +249,20 @@ func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { return } +func (self *RdDbChan) GetGroupOfCart(cart uint) (string, error) { + res_ch := make(chan getGroupOfCartResult) + req := getGroupOfCartRequest{} + req.cart = cart + req.response = res_ch + self.getGroupOfCartChan <- req + + res := <-res_ch + if res.err != nil { + return "", res.err + } + return res.group, nil +} + func (self *RdDb) getLogTableName(log string) (logtable string, err error) { logtable = strings.Replace(log, " ", "_", -1) + "_LOG" if !mysqlTableNameRe.MatchString(logtable) { @@ -280,6 +309,20 @@ func (self *RdDb) getShowInfo(showid uint) (result getShowInfoResult) { return } +func (self *RdDbChan) GetShowInfo(showid uint) (string, int, int, []uint, error) { + res_ch := make(chan getShowInfoResult) + req := getShowInfoRequest{} + req.showid = showid + req.response = res_ch + self.getShowInfoChan <- req + + res := <-res_ch + if res.err != nil { + return "", 0, 0, nil, res.err + } + return res.group, res.norm_lvl, res.trim_lvl, res.carts, nil +} + func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { var cnt int if result.err = self.checkMusicGroupStmt.QueryRow(group).Scan(&cnt); result.err != nil { @@ -293,6 +336,20 @@ func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { return } +func (self *RdDbChan) CheckMusicGroup(groupname string) (bool, error) { + res_ch := make(chan checkMusicGroupResult) + req := checkMusicGroupRequest{} + req.group = groupname + req.response = res_ch + self.checkMusicGroupChan <- req + + res := <-res_ch + if res.err != nil { + return false, res.err + } + return res.ismusic, nil +} + func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&result.norm_lvl, &result.trim_lvl) if result.err != nil { @@ -304,6 +361,20 @@ func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { return } +func (self *RdDbChan) GetMusicInfo(groupname string) (int, int, error) { + res_ch := make(chan getMusicInfoResult) + req := getMusicInfoRequest{} + req.group = groupname + req.response = res_ch + self.getMusicInfoChan <- req + + res := <-res_ch + if res.err != nil { + return 0, 0, res.err + } + return res.norm_lvl, res.trim_lvl, nil +} + func (self *RdDb) dispatchRequests() { defer func() { self.done <- true }() for { -- cgit v0.10.2 From e40f502673a130c7d1a9c9b3ffa823399a4cd3cf Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 28 Dec 2015 16:55:01 +0100 Subject: moved ImporteContext stuff to core.go diff --git a/core.go b/core.go index 5498499..3b8280a 100644 --- a/core.go +++ b/core.go @@ -26,14 +26,21 @@ package rhimport import ( // "io/ioutil" + "fmt" "github.com/golang-basic/go-curl" "log" "os" ) +const ( + CART_MAX = 999999 + CUT_MAX = 999 +) + var ( - rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) - rhdl = log.New(os.Stderr, "[rhimport-dbg]\t", log.LstdFlags) + bool2str = map[bool]string{false: "0", true: "1"} + rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) + rhdl = log.New(os.Stderr, "[rhimport-dbg]\t", log.LstdFlags) //rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) ) @@ -41,3 +48,130 @@ func init() { curl.GlobalInit(curl.GLOBAL_ALL) fetcher_init() } + +type ImportProgressCB func(step int, step_name string, progress float64, userdata interface{}) bool + +type ImportContext struct { + conf *Config + rddb *RdDbChan + UserName string + Password string + Trusted bool + ShowId uint + ClearShowCarts bool + GroupName string + Cart uint + ClearCart bool + Cut uint + Channels uint + NormalizationLevel int + AutotrimLevel int + UseMetaData bool + SourceUri string + SourceFile string + DeleteSourceFile bool + DeleteSourceDir bool + ProgressCallBack ImportProgressCB + ProgressCallBackData interface{} + Cancel <-chan bool +} + +func NewImportContext(conf *Config, rddb *RdDbChan, user string) *ImportContext { + ctx := new(ImportContext) + ctx.conf = conf + ctx.rddb = rddb + ctx.UserName = user + ctx.Password = "" + ctx.Trusted = false + ctx.ShowId = 0 + ctx.ClearShowCarts = false + ctx.GroupName = "" + ctx.Cart = 0 + ctx.ClearCart = false + ctx.Cut = 0 + ctx.Channels = conf.ImportParamDefaults.Channels + ctx.NormalizationLevel = conf.ImportParamDefaults.NormalizationLevel + ctx.AutotrimLevel = conf.ImportParamDefaults.AutotrimLevel + ctx.UseMetaData = conf.ImportParamDefaults.UseMetaData + ctx.SourceFile = "" + ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false + ctx.ProgressCallBack = nil + ctx.Cancel = nil + + return ctx +} + +func (ctx *ImportContext) SanityCheck() error { + if ctx.UserName == "" { + return fmt.Errorf("empty Username is not allowed") + } + if ctx.Password == "" && !ctx.Trusted { + return fmt.Errorf("empty Password on untrusted control interface is not allowed") + } + if ctx.ShowId != 0 { + if ctx.ShowId != 0 && ctx.ShowId > CART_MAX { + return fmt.Errorf("ShowId %d is outside of allowed range (0 < show-id < %d)", ctx.ShowId, CART_MAX) + } + if ctx.Cart != 0 && ctx.Cart > CART_MAX { + return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) + } + return nil + } + if ctx.GroupName != "" { + ismusic, err := ctx.checkMusicGroup() + if err != nil { + return err + } + if !ismusic { + return fmt.Errorf("supplied GroupName '%s' is not a music pool", ctx.GroupName) + } + if ctx.Cart != 0 || ctx.Cut != 0 { + return fmt.Errorf("Cart and Cut must not be supplied when importing into a music group") + } + return nil + } + if ctx.Cart == 0 { + return fmt.Errorf("either ShowId, PoolName or CartNumber must be supplied") + } + if ctx.Cart > CART_MAX { + return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) + } + if ctx.Cut != 0 && ctx.Cut > CUT_MAX { + return fmt.Errorf("Cut %d is outside of allowed range (0 < cart < %d)", ctx.Cut, CUT_MAX) + } + if ctx.Channels != 1 && ctx.Channels != 2 { + return fmt.Errorf("channles must be 1 or 2") + } + return nil +} + +func (ctx *ImportContext) getPassword(cached bool) (err error) { + ctx.Password, err = ctx.rddb.GetPassword(ctx.UserName, cached) + return +} + +func (ctx *ImportContext) getGroupOfCart() (err error) { + ctx.GroupName, err = ctx.rddb.GetGroupOfCart(ctx.Cart) + return +} + +func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { + ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.rddb.GetShowInfo(ctx.ShowId) + ctx.Channels = 2 + ctx.UseMetaData = true + return +} + +func (ctx *ImportContext) checkMusicGroup() (bool, error) { + return ctx.rddb.CheckMusicGroup(ctx.GroupName) +} + +func (ctx *ImportContext) getMusicInfo() (err error) { + ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.rddb.GetMusicInfo(ctx.GroupName) + ctx.Channels = 2 + ctx.UseMetaData = true + ctx.Cart = 0 + ctx.Cut = 0 + return +} diff --git a/importer.go b/importer.go index 28ca55a..406c58e 100644 --- a/importer.go +++ b/importer.go @@ -35,142 +35,6 @@ import ( "path" ) -const ( - CART_MAX = 999999 - CUT_MAX = 999 -) - -var ( - bool2str = map[bool]string{false: "0", true: "1"} -) - -type ImportProgressCB func(step int, step_name string, progress float64, userdata interface{}) bool - -type ImportContext struct { - conf *Config - rddb *RdDbChan - UserName string - Password string - Trusted bool - ShowId uint - ClearShowCarts bool - GroupName string - Cart uint - ClearCart bool - Cut uint - Channels uint - NormalizationLevel int - AutotrimLevel int - UseMetaData bool - SourceUri string - SourceFile string - DeleteSourceFile bool - DeleteSourceDir bool - ProgressCallBack ImportProgressCB - ProgressCallBackData interface{} - Cancel <-chan bool -} - -func NewImportContext(conf *Config, rddb *RdDbChan, user string) *ImportContext { - ctx := new(ImportContext) - ctx.conf = conf - ctx.rddb = rddb - ctx.UserName = user - ctx.Password = "" - ctx.Trusted = false - ctx.ShowId = 0 - ctx.ClearShowCarts = false - ctx.GroupName = "" - ctx.Cart = 0 - ctx.ClearCart = false - ctx.Cut = 0 - ctx.Channels = conf.ImportParamDefaults.Channels - ctx.NormalizationLevel = conf.ImportParamDefaults.NormalizationLevel - ctx.AutotrimLevel = conf.ImportParamDefaults.AutotrimLevel - ctx.UseMetaData = conf.ImportParamDefaults.UseMetaData - ctx.SourceFile = "" - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - ctx.ProgressCallBack = nil - ctx.Cancel = nil - - return ctx -} - -func (ctx *ImportContext) SanityCheck() error { - if ctx.UserName == "" { - return fmt.Errorf("empty Username is not allowed") - } - if ctx.Password == "" && !ctx.Trusted { - return fmt.Errorf("empty Password on untrusted control interface is not allowed") - } - if ctx.ShowId != 0 { - if ctx.ShowId != 0 && ctx.ShowId > CART_MAX { - return fmt.Errorf("ShowId %d is outside of allowed range (0 < show-id < %d)", ctx.ShowId, CART_MAX) - } - if ctx.Cart != 0 && ctx.Cart > CART_MAX { - return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) - } - return nil - } - if ctx.GroupName != "" { - ismusic, err := ctx.checkMusicGroup() - if err != nil { - return err - } - if !ismusic { - return fmt.Errorf("supplied GroupName '%s' is not a music pool", ctx.GroupName) - } - if ctx.Cart != 0 || ctx.Cut != 0 { - return fmt.Errorf("Cart and Cut must not be supplied when importing into a music group") - } - return nil - } - if ctx.Cart == 0 { - return fmt.Errorf("either ShowId, PoolName or CartNumber must be supplied") - } - if ctx.Cart > CART_MAX { - return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) - } - if ctx.Cut != 0 && ctx.Cut > CUT_MAX { - return fmt.Errorf("Cut %d is outside of allowed range (0 < cart < %d)", ctx.Cut, CUT_MAX) - } - if ctx.Channels != 1 && ctx.Channels != 2 { - return fmt.Errorf("channles must be 1 or 2") - } - return nil -} - -func (ctx *ImportContext) getPassword(cached bool) (err error) { - ctx.Password, err = ctx.rddb.GetPassword(ctx.UserName, cached) - return -} - -func (ctx *ImportContext) getGroupOfCart() (err error) { - ctx.GroupName, err = ctx.rddb.GetGroupOfCart(ctx.Cart) - return -} - -func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { - ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.rddb.GetShowInfo(ctx.ShowId) - ctx.Channels = 2 - ctx.UseMetaData = true - return -} - -func (ctx *ImportContext) checkMusicGroup() (bool, error) { - return ctx.rddb.CheckMusicGroup(ctx.GroupName) -} - -func (ctx *ImportContext) getMusicInfo() (err error) { - ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.rddb.GetMusicInfo(ctx.GroupName) - ctx.Channels = 2 - ctx.UseMetaData = true - ctx.Cart = 0 - ctx.Cut = 0 - return -} - type ImportResult struct { ResponseCode int ErrorString string -- cgit v0.10.2 From 4fb651e90364cf822007ef16f4190ad853f04166 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 28 Dec 2015 17:10:06 +0100 Subject: refactoring of rddb, seperate public from private interface diff --git a/rddb.go b/rddb.go index ead1ac0..8bec3c0 100644 --- a/rddb.go +++ b/rddb.go @@ -97,14 +97,6 @@ type getMusicInfoRequest struct { response chan<- getMusicInfoResult } -type RdDbChan struct { - getPasswordChan chan<- getPasswordRequest - getGroupOfCartChan chan<- getGroupOfCartRequest - getShowInfoChan chan<- getShowInfoRequest - checkMusicGroupChan chan<- checkMusicGroupRequest - getMusicInfoChan chan<- getMusicInfoRequest -} - type RdDb struct { dbh *sql.DB password_cache map[string]string @@ -177,48 +169,6 @@ func (self *RdDb) getPassword(username string, cached bool) (result getPasswordR return } -func (self *RdDbChan) GetPassword(username string, cached bool) (string, error) { - res_ch := make(chan getPasswordResult) - req := getPasswordRequest{} - req.username = username - req.cached = cached - req.response = res_ch - self.getPasswordChan <- req - - res := <-res_ch - if res.err != nil { - return "", res.err - } - return res.password, nil -} - -func (self *RdDbChan) CheckPassword(username, password string) (bool, error) { - cached := true - - for { - res_ch := make(chan getPasswordResult) - req := getPasswordRequest{} - req.username = username - req.cached = cached - req.response = res_ch - self.getPasswordChan <- req - - res := <-res_ch - if res.err != nil { - return false, res.err - } - if password == res.password { - return true, nil - } - if cached { - cached = false - } else { - break - } - } - return false, nil -} - func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { var rows *sql.Rows if rows, result.err = self.getGroupOfCartStmt.Query(cart, cart); result.err != nil { @@ -249,20 +199,6 @@ func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { return } -func (self *RdDbChan) GetGroupOfCart(cart uint) (string, error) { - res_ch := make(chan getGroupOfCartResult) - req := getGroupOfCartRequest{} - req.cart = cart - req.response = res_ch - self.getGroupOfCartChan <- req - - res := <-res_ch - if res.err != nil { - return "", res.err - } - return res.group, nil -} - func (self *RdDb) getLogTableName(log string) (logtable string, err error) { logtable = strings.Replace(log, " ", "_", -1) + "_LOG" if !mysqlTableNameRe.MatchString(logtable) { @@ -309,20 +245,6 @@ func (self *RdDb) getShowInfo(showid uint) (result getShowInfoResult) { return } -func (self *RdDbChan) GetShowInfo(showid uint) (string, int, int, []uint, error) { - res_ch := make(chan getShowInfoResult) - req := getShowInfoRequest{} - req.showid = showid - req.response = res_ch - self.getShowInfoChan <- req - - res := <-res_ch - if res.err != nil { - return "", 0, 0, nil, res.err - } - return res.group, res.norm_lvl, res.trim_lvl, res.carts, nil -} - func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { var cnt int if result.err = self.checkMusicGroupStmt.QueryRow(group).Scan(&cnt); result.err != nil { @@ -336,20 +258,6 @@ func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { return } -func (self *RdDbChan) CheckMusicGroup(groupname string) (bool, error) { - res_ch := make(chan checkMusicGroupResult) - req := checkMusicGroupRequest{} - req.group = groupname - req.response = res_ch - self.checkMusicGroupChan <- req - - res := <-res_ch - if res.err != nil { - return false, res.err - } - return res.ismusic, nil -} - func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&result.norm_lvl, &result.trim_lvl) if result.err != nil { @@ -361,20 +269,6 @@ func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { return } -func (self *RdDbChan) GetMusicInfo(groupname string) (int, int, error) { - res_ch := make(chan getMusicInfoResult) - req := getMusicInfoRequest{} - req.group = groupname - req.response = res_ch - self.getMusicInfoChan <- req - - res := <-res_ch - if res.err != nil { - return 0, 0, res.err - } - return res.norm_lvl, res.trim_lvl, nil -} - func (self *RdDb) dispatchRequests() { defer func() { self.done <- true }() for { @@ -395,6 +289,115 @@ func (self *RdDb) dispatchRequests() { } } +// ********************************************************* +// Public Interface + +type RdDbChan struct { + getPasswordChan chan<- getPasswordRequest + getGroupOfCartChan chan<- getGroupOfCartRequest + getShowInfoChan chan<- getShowInfoRequest + checkMusicGroupChan chan<- checkMusicGroupRequest + getMusicInfoChan chan<- getMusicInfoRequest +} + +func (self *RdDbChan) GetPassword(username string, cached bool) (string, error) { + res_ch := make(chan getPasswordResult) + req := getPasswordRequest{} + req.username = username + req.cached = cached + req.response = res_ch + self.getPasswordChan <- req + + res := <-res_ch + if res.err != nil { + return "", res.err + } + return res.password, nil +} + +func (self *RdDbChan) CheckPassword(username, password string) (bool, error) { + cached := true + + for { + res_ch := make(chan getPasswordResult) + req := getPasswordRequest{} + req.username = username + req.cached = cached + req.response = res_ch + self.getPasswordChan <- req + + res := <-res_ch + if res.err != nil { + return false, res.err + } + if password == res.password { + return true, nil + } + if cached { + cached = false + } else { + break + } + } + return false, nil +} + +func (self *RdDbChan) GetGroupOfCart(cart uint) (string, error) { + res_ch := make(chan getGroupOfCartResult) + req := getGroupOfCartRequest{} + req.cart = cart + req.response = res_ch + self.getGroupOfCartChan <- req + + res := <-res_ch + if res.err != nil { + return "", res.err + } + return res.group, nil +} + +func (self *RdDbChan) GetShowInfo(showid uint) (string, int, int, []uint, error) { + res_ch := make(chan getShowInfoResult) + req := getShowInfoRequest{} + req.showid = showid + req.response = res_ch + self.getShowInfoChan <- req + + res := <-res_ch + if res.err != nil { + return "", 0, 0, nil, res.err + } + return res.group, res.norm_lvl, res.trim_lvl, res.carts, nil +} + +func (self *RdDbChan) CheckMusicGroup(groupname string) (bool, error) { + res_ch := make(chan checkMusicGroupResult) + req := checkMusicGroupRequest{} + req.group = groupname + req.response = res_ch + self.checkMusicGroupChan <- req + + res := <-res_ch + if res.err != nil { + return false, res.err + } + return res.ismusic, nil +} + +func (self *RdDbChan) GetMusicInfo(groupname string) (int, int, error) { + res_ch := make(chan getMusicInfoResult) + req := getMusicInfoRequest{} + req.group = groupname + req.response = res_ch + self.getMusicInfoChan <- req + + res := <-res_ch + if res.err != nil { + return 0, 0, res.err + } + return res.norm_lvl, res.trim_lvl, nil +} + func (self *RdDb) GetInterface() *RdDbChan { ch := &RdDbChan{} ch.getPasswordChan = self.getPasswordChan -- cgit v0.10.2 From 1c54091cf71bf9f92475e02b94153e8b4bfbe36b Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 28 Dec 2015 17:20:14 +0100 Subject: refactoring of session-store, seperate public from private interface diff --git a/session_store.go b/session_store.go index e065182..43b5d23 100644 --- a/session_store.go +++ b/session_store.go @@ -63,12 +63,6 @@ type removeSessionRequest struct { response chan removeSessionResponse } -type SessionStoreChan struct { - newChan chan<- newSessionRequest - getChan chan<- getSessionRequest - removeChan chan<- removeSessionRequest -} - type SessionStore struct { store map[string]map[string]*Session quit chan bool @@ -99,20 +93,6 @@ func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { return } -func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, error) { - res_ch := make(chan newSessionResponse) - req := newSessionRequest{} - req.ctx = ctx - req.response = res_ch - self.newChan <- req - - res := <-res_ch - if res.err != nil { - return "", nil, res.err - } - return res.id, res.session, nil -} - func (self *SessionStore) get(user, id string) (resp getSessionResponse) { if session, exists := self.store[user][id]; exists { resp.session = session.getInterface() @@ -122,21 +102,6 @@ func (self *SessionStore) get(user, id string) (resp getSessionResponse) { return } -func (self *SessionStoreChan) Get(user, id string) (*SessionChan, error) { - res_ch := make(chan getSessionResponse) - req := getSessionRequest{} - req.user = user - req.id = id - req.response = res_ch - self.getChan <- req - - res := <-res_ch - if res.err != nil { - return nil, res.err - } - return res.session, nil -} - func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { if session, exists := self.store[user][id]; exists { go session.Cleanup() // cleanup could take a while -> don't block all the other stuff @@ -154,18 +119,6 @@ func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { return } -func (self *SessionStoreChan) Remove(user, id string) error { - res_ch := make(chan removeSessionResponse) - req := removeSessionRequest{} - req.user = user - req.id = id - req.response = res_ch - self.removeChan <- req - - res := <-res_ch - return res.err -} - func (self *SessionStore) dispatchRequests() { defer func() { self.done <- true }() for { @@ -182,6 +135,56 @@ func (self *SessionStore) dispatchRequests() { } } +// ********************************************************* +// Public Interface + +type SessionStoreChan struct { + newChan chan<- newSessionRequest + getChan chan<- getSessionRequest + removeChan chan<- removeSessionRequest +} + +func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, error) { + res_ch := make(chan newSessionResponse) + req := newSessionRequest{} + req.ctx = ctx + req.response = res_ch + self.newChan <- req + + res := <-res_ch + if res.err != nil { + return "", nil, res.err + } + return res.id, res.session, nil +} + +func (self *SessionStoreChan) Get(user, id string) (*SessionChan, error) { + res_ch := make(chan getSessionResponse) + req := getSessionRequest{} + req.user = user + req.id = id + req.response = res_ch + self.getChan <- req + + res := <-res_ch + if res.err != nil { + return nil, res.err + } + return res.session, nil +} + +func (self *SessionStoreChan) Remove(user, id string) error { + res_ch := make(chan removeSessionResponse) + req := removeSessionRequest{} + req.user = user + req.id = id + req.response = res_ch + self.removeChan <- req + + res := <-res_ch + return res.err +} + func (self *SessionStore) GetInterface() *SessionStoreChan { ch := &SessionStoreChan{} ch.newChan = self.newChan -- cgit v0.10.2 From 491fbeb2408b7c37edb8067d51421a98cc1db3c8 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 28 Dec 2015 17:21:57 +0100 Subject: refactoring of session, seperate public from private interface diff --git a/session.go b/session.go index 88648f7..71251f1 100644 --- a/session.go +++ b/session.go @@ -38,13 +38,6 @@ const ( SESSION_TIMEOUT ) -type SessionChan struct { - runChan chan<- time.Duration - cancelChan chan<- bool - addProgressChan chan<- sessionAddProgressHandlerRequest - addDoneChan chan<- sessionAddDoneHandlerRequest -} - type Session struct { ctx ImportContext state int @@ -138,13 +131,6 @@ func (self *Session) run(timeout time.Duration) { return } -func (self *SessionChan) Run(timeout time.Duration) { - select { - case self.runChan <- timeout: - default: // command is already pending or session is about to be closed/removed - } -} - func (self *Session) cancel() { rhdl.Println("Session: canceling running import") select { @@ -154,13 +140,6 @@ func (self *Session) cancel() { self.state = SESSION_CANCELED } -func (self *SessionChan) Cancel() { - select { - case self.cancelChan <- true: - default: // cancel is already pending or session is about to be closed/removed - } -} - func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressCB) (resp sessionAddProgressHandlerResponse) { if self.state != SESSION_NEW && self.state != SESSION_RUNNING { resp.err = fmt.Errorf("session is already done/canceled") @@ -169,22 +148,6 @@ func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressC return } -func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgressCB) error { - res_ch := make(chan sessionAddProgressHandlerResponse) - req := sessionAddProgressHandlerRequest{} - req.userdata = userdata - req.callback = cb - req.response = res_ch - select { - case self.addProgressChan <- req: - default: - return fmt.Errorf("session is about to be closed/removed") - } - - res := <-res_ch - return res.err -} - func (self *Session) addDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) (resp sessionAddDoneHandlerResponse) { if self.state != SESSION_NEW && self.state != SESSION_RUNNING { resp.err = fmt.Errorf("session is already done/canceled") @@ -203,22 +166,6 @@ func (self *Session) callProgressHandler(p *ProgressData) { } } -func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) error { - res_ch := make(chan sessionAddDoneHandlerResponse) - req := sessionAddDoneHandlerRequest{} - req.userdata = userdata - req.callback = cb - req.response = res_ch - select { - case self.addDoneChan <- req: - default: - return fmt.Errorf("session is about to be closed/removed") - } - - res := <-res_ch - return res.err -} - func (self *Session) callDoneHandler(r *ImportResult) { for _, cb := range self.doneCBs { if cb.cb != nil { @@ -275,6 +222,62 @@ func (self *Session) dispatchRequests() { } } +// ********************************************************* +// Public Interface + +type SessionChan struct { + runChan chan<- time.Duration + cancelChan chan<- bool + addProgressChan chan<- sessionAddProgressHandlerRequest + addDoneChan chan<- sessionAddDoneHandlerRequest +} + +func (self *SessionChan) Run(timeout time.Duration) { + select { + case self.runChan <- timeout: + default: // command is already pending or session is about to be closed/removed + } +} + +func (self *SessionChan) Cancel() { + select { + case self.cancelChan <- true: + default: // cancel is already pending or session is about to be closed/removed + } +} + +func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgressCB) error { + res_ch := make(chan sessionAddProgressHandlerResponse) + req := sessionAddProgressHandlerRequest{} + req.userdata = userdata + req.callback = cb + req.response = res_ch + select { + case self.addProgressChan <- req: + default: + return fmt.Errorf("session is about to be closed/removed") + } + + res := <-res_ch + return res.err +} + +func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) error { + res_ch := make(chan sessionAddDoneHandlerResponse) + req := sessionAddDoneHandlerRequest{} + req.userdata = userdata + req.callback = cb + req.response = res_ch + select { + case self.addDoneChan <- req: + default: + return fmt.Errorf("session is about to be closed/removed") + } + + res := <-res_ch + return res.err +} + func (self *Session) getInterface() *SessionChan { ch := &SessionChan{} ch.runChan = self.runChan -- cgit v0.10.2 From 3a6efe6d413c24b914d2b7889eb520b3f1a0820f Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Mon, 28 Dec 2015 17:34:51 +0100 Subject: minor documenation improvement diff --git a/importer.go b/importer.go index 406c58e..abc99e8 100644 --- a/importer.go +++ b/importer.go @@ -455,7 +455,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { rmCartOnErr := false rmCutOnErr := false res = &ImportResult{ResponseCode: http.StatusOK} - if ctx.ShowId != 0 { + if ctx.ShowId != 0 { // Import to a show var show_carts []uint if show_carts, err = ctx.getShowInfo(); err != nil { return @@ -476,7 +476,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } } rmCartOnErr = true - } else if ctx.GroupName != "" { + } else if ctx.GroupName != "" { // Import to music pool if err = ctx.getMusicInfo(); err != nil { return } @@ -484,7 +484,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { return } rmCartOnErr = true - } else if ctx.Cart != 0 && ctx.Cut == 0 { + } else if ctx.Cart != 0 && ctx.Cut == 0 { // Import to Cart if ctx.ClearCart { if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return @@ -505,7 +505,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } } - if ctx.Cart != 0 && ctx.Cut != 0 { + if ctx.Cart != 0 && ctx.Cut != 0 { // Import to specific Cut within Cart if err = import_audio(ctx, res); err != nil || res.ResponseCode != http.StatusOK { if err != nil { rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", ctx.Cart, ctx.Cut, err) -- cgit v0.10.2 From 46ae2f5c6db5897c00d75ce8d29cc0ad55b7b613 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 29 Dec 2015 02:38:25 +0100 Subject: improved error handling for session store diff --git a/session.go b/session.go index 71251f1..94ce66a 100644 --- a/session.go +++ b/session.go @@ -190,7 +190,7 @@ func (self *Session) dispatchRequests() { self.cancel() } self.state = SESSION_TIMEOUT - r := &ImportResult{500, "timeout", 0, 0} + r := &ImportResult{500, "session timed out", 0, 0} self.callDoneHandler(r) if self.removeFunc != nil { self.removeFunc() diff --git a/session_store.go b/session_store.go index 43b5d23..b7ea3d9 100644 --- a/session_store.go +++ b/session_store.go @@ -28,13 +28,15 @@ import ( "encoding/base32" "fmt" "github.com/satori/go.uuid" + "net/http" "strings" ) type newSessionResponse struct { - id string - session *SessionChan - err error + id string + session *SessionChan + responsecode int + errorstring string } type newSessionRequest struct { @@ -43,8 +45,9 @@ type newSessionRequest struct { } type getSessionResponse struct { - session *SessionChan - err error + session *SessionChan + responsecode int + errorstring string } type getSessionRequest struct { @@ -54,7 +57,8 @@ type getSessionRequest struct { } type removeSessionResponse struct { - err error + responsecode int + errorstring string } type removeSessionRequest struct { @@ -73,12 +77,16 @@ type SessionStore struct { } func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" if !ctx.Trusted { if ok, err := ctx.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { - resp.err = err + resp.responsecode = http.StatusInternalServerError + resp.errorstring = err.Error() return } else if !ok { - resp.err = fmt.Errorf("invalid username and/or password") + resp.responsecode = http.StatusUnauthorized + resp.errorstring = "invalid username and/or password" return } } @@ -94,15 +102,20 @@ func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { } func (self *SessionStore) get(user, id string) (resp getSessionResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" if session, exists := self.store[user][id]; exists { resp.session = session.getInterface() } else { - resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + resp.responsecode = http.StatusNotFound + resp.errorstring = fmt.Sprintf("SessionStore: session '%s/%s' not found", user, id) } return } func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" if session, exists := self.store[user][id]; exists { go session.Cleanup() // cleanup could take a while -> don't block all the other stuff delete(self.store[user], id) @@ -114,7 +127,8 @@ func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { } } } else { - resp.err = fmt.Errorf("SessionStore: session '%s/%s' not found", user, id) + resp.responsecode = http.StatusNotFound + resp.errorstring = fmt.Sprintf("SessionStore: session '%s/%s' not found", user, id) } return } @@ -144,7 +158,7 @@ type SessionStoreChan struct { removeChan chan<- removeSessionRequest } -func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, error) { +func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, int, string) { res_ch := make(chan newSessionResponse) req := newSessionRequest{} req.ctx = ctx @@ -152,13 +166,10 @@ func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, err self.newChan <- req res := <-res_ch - if res.err != nil { - return "", nil, res.err - } - return res.id, res.session, nil + return res.id, res.session, res.responsecode, res.errorstring } -func (self *SessionStoreChan) Get(user, id string) (*SessionChan, error) { +func (self *SessionStoreChan) Get(user, id string) (*SessionChan, int, string) { res_ch := make(chan getSessionResponse) req := getSessionRequest{} req.user = user @@ -167,13 +178,10 @@ func (self *SessionStoreChan) Get(user, id string) (*SessionChan, error) { self.getChan <- req res := <-res_ch - if res.err != nil { - return nil, res.err - } - return res.session, nil + return res.session, res.responsecode, res.errorstring } -func (self *SessionStoreChan) Remove(user, id string) error { +func (self *SessionStoreChan) Remove(user, id string) (int, string) { res_ch := make(chan removeSessionResponse) req := removeSessionRequest{} req.user = user @@ -182,7 +190,7 @@ func (self *SessionStoreChan) Remove(user, id string) error { self.removeChan <- req res := <-res_ch - return res.err + return res.responsecode, res.errorstring } func (self *SessionStore) GetInterface() *SessionStoreChan { -- cgit v0.10.2 From 43d7fc0b4110ebf27c2847fc2660916da60629e9 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 29 Dec 2015 04:04:07 +0100 Subject: improved session id generation with less deps diff --git a/session_store.go b/session_store.go index b7ea3d9..5025d8c 100644 --- a/session_store.go +++ b/session_store.go @@ -25,11 +25,10 @@ package rhimport import ( - "encoding/base32" + "crypto/rand" + "encoding/base64" "fmt" - "github.com/satori/go.uuid" "net/http" - "strings" ) type newSessionResponse struct { @@ -76,6 +75,14 @@ type SessionStore struct { removeChan chan removeSessionRequest } +func generateSessionId() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return base64.RawStdEncoding.EncodeToString(b[:]), nil +} + func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" @@ -90,14 +97,18 @@ func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { return } } - b := uuid.NewV4().Bytes() - resp.id = strings.ToLower(strings.TrimRight(base32.StdEncoding.EncodeToString(b), "=")) - if _, exists := self.store[ctx.UserName]; !exists { - self.store[ctx.UserName] = make(map[string]*Session) + if id, err := generateSessionId(); err != nil { + resp.responsecode = http.StatusInternalServerError + resp.errorstring = err.Error() + } else { + resp.id = id + if _, exists := self.store[ctx.UserName]; !exists { + self.store[ctx.UserName] = make(map[string]*Session) + } + self.store[ctx.UserName][resp.id] = NewSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }) + resp.session = self.store[ctx.UserName][resp.id].getInterface() + rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) } - self.store[ctx.UserName][resp.id] = NewSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }) - resp.session = self.store[ctx.UserName][resp.id].getInterface() - rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) return } -- cgit v0.10.2 From f46281a93faf72331178a93ebf67cfd23fa81395 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 30 Dec 2015 20:23:52 +0100 Subject: some cleanup diff --git a/session.go b/session.go index 94ce66a..36e7938 100644 --- a/session.go +++ b/session.go @@ -67,9 +67,9 @@ type SessionDoneCB struct { } type ProgressData struct { - step int - step_name string - progress float64 + Step int + StepName string + Progress float64 } type sessionAddProgressHandlerResponse struct { @@ -159,7 +159,7 @@ func (self *Session) addDoneHandler(userdata interface{}, cb func(ImportResult, func (self *Session) callProgressHandler(p *ProgressData) { for _, cb := range self.progressCBs { if cb.cb != nil { - if keep := cb.cb(p.step, p.step_name, p.progress, cb.userdata); !keep { + if keep := cb.cb(p.Step, p.StepName, p.Progress, cb.userdata); !keep { cb.cb = nil } } -- cgit v0.10.2 From 6ff5e0a94f38b4501ffdc7ff51ff6766d03c5a65 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 30 Dec 2015 20:58:28 +0100 Subject: session store and web socket interface now support to set a reference id diff --git a/session_store.go b/session_store.go index 5025d8c..7fe6585 100644 --- a/session_store.go +++ b/session_store.go @@ -40,11 +40,13 @@ type newSessionResponse struct { type newSessionRequest struct { ctx *ImportContext + refId string response chan newSessionResponse } type getSessionResponse struct { session *SessionChan + refId string responsecode int errorstring string } @@ -52,6 +54,7 @@ type getSessionResponse struct { type getSessionRequest struct { user string id string + refId string response chan getSessionResponse } @@ -66,8 +69,13 @@ type removeSessionRequest struct { response chan removeSessionResponse } +type SessionStoreElement struct { + s *Session + refId string +} + type SessionStore struct { - store map[string]map[string]*Session + store map[string]map[string]*SessionStoreElement quit chan bool done chan bool newChan chan newSessionRequest @@ -83,7 +91,7 @@ func generateSessionId() (string, error) { return base64.RawStdEncoding.EncodeToString(b[:]), nil } -func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { +func (self *SessionStore) new(ctx *ImportContext, refId string) (resp newSessionResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" if !ctx.Trusted { @@ -103,10 +111,11 @@ func (self *SessionStore) new(ctx *ImportContext) (resp newSessionResponse) { } else { resp.id = id if _, exists := self.store[ctx.UserName]; !exists { - self.store[ctx.UserName] = make(map[string]*Session) + self.store[ctx.UserName] = make(map[string]*SessionStoreElement) } - self.store[ctx.UserName][resp.id] = NewSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }) - resp.session = self.store[ctx.UserName][resp.id].getInterface() + s := &SessionStoreElement{NewSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }), refId} + self.store[ctx.UserName][resp.id] = s + resp.session = self.store[ctx.UserName][resp.id].s.getInterface() rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) } return @@ -116,7 +125,8 @@ func (self *SessionStore) get(user, id string) (resp getSessionResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" if session, exists := self.store[user][id]; exists { - resp.session = session.getInterface() + resp.session = session.s.getInterface() + resp.refId = session.refId } else { resp.responsecode = http.StatusNotFound resp.errorstring = fmt.Sprintf("SessionStore: session '%s/%s' not found", user, id) @@ -128,7 +138,7 @@ func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" if session, exists := self.store[user][id]; exists { - go session.Cleanup() // cleanup could take a while -> don't block all the other stuff + go session.s.Cleanup() // cleanup could take a while -> don't block all the other stuff delete(self.store[user], id) rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) if userstore, exists := self.store[user]; exists { @@ -151,7 +161,7 @@ func (self *SessionStore) dispatchRequests() { case <-self.quit: return case req := <-self.newChan: - req.response <- self.new(req.ctx) + req.response <- self.new(req.ctx, req.refId) case req := <-self.getChan: req.response <- self.get(req.user, req.id) case req := <-self.removeChan: @@ -169,10 +179,11 @@ type SessionStoreChan struct { removeChan chan<- removeSessionRequest } -func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, int, string) { +func (self *SessionStoreChan) New(ctx *ImportContext, refId string) (string, *SessionChan, int, string) { res_ch := make(chan newSessionResponse) req := newSessionRequest{} req.ctx = ctx + req.refId = refId req.response = res_ch self.newChan <- req @@ -180,7 +191,7 @@ func (self *SessionStoreChan) New(ctx *ImportContext) (string, *SessionChan, int return res.id, res.session, res.responsecode, res.errorstring } -func (self *SessionStoreChan) Get(user, id string) (*SessionChan, int, string) { +func (self *SessionStoreChan) Get(user, id string) (*SessionChan, string, int, string) { res_ch := make(chan getSessionResponse) req := getSessionRequest{} req.user = user @@ -189,7 +200,7 @@ func (self *SessionStoreChan) Get(user, id string) (*SessionChan, int, string) { self.getChan <- req res := <-res_ch - return res.session, res.responsecode, res.errorstring + return res.session, res.refId, res.responsecode, res.errorstring } func (self *SessionStoreChan) Remove(user, id string) (int, string) { @@ -227,7 +238,7 @@ func NewSessionStore(conf *Config) (store *SessionStore, err error) { store.quit = make(chan bool) store.done = make(chan bool) - store.store = make(map[string]map[string]*Session) + store.store = make(map[string]map[string]*SessionStoreElement) store.newChan = make(chan newSessionRequest, 10) store.getChan = make(chan getSessionRequest, 10) store.removeChan = make(chan removeSessionRequest, 10) -- cgit v0.10.2 From 75a9a00e87da7bc4bb6990330ecb6b217d98de82 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 30 Dec 2015 21:50:46 +0100 Subject: improved response handling at websocket interface diff --git a/session_store.go b/session_store.go index 7fe6585..0d149c6 100644 --- a/session_store.go +++ b/session_store.go @@ -58,6 +58,19 @@ type getSessionRequest struct { response chan getSessionResponse } +type listSessionsResponse struct { + sessions map[string]string + responsecode int + errorstring string +} + +type listSessionsRequest struct { + user string + password string + trusted bool + response chan listSessionsResponse +} + type removeSessionResponse struct { responsecode int errorstring string @@ -80,6 +93,7 @@ type SessionStore struct { done chan bool newChan chan newSessionRequest getChan chan getSessionRequest + listChan chan listSessionsRequest removeChan chan removeSessionRequest } @@ -134,6 +148,30 @@ func (self *SessionStore) get(user, id string) (resp getSessionResponse) { return } +func (self *SessionStore) list(user, password string, trusted bool) (resp listSessionsResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" + // TODO: enable this check as soon as the session store handles the rddb itself + // if !trusted { + // if ok, err := self.rddb.CheckPassword(user, password); err != nil { + // resp.responsecode = http.StatusInternalServerError + // resp.errorstring = err.Error() + // return + // } else if !ok { + // resp.responsecode = http.StatusUnauthorized + // resp.errorstring = "invalid username and/or password" + // return + // } + // } + resp.sessions = make(map[string]string) + if sessions, exists := self.store[user]; exists { + for id, e := range sessions { + resp.sessions[id] = e.refId + } + } + return +} + func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" @@ -164,6 +202,8 @@ func (self *SessionStore) dispatchRequests() { req.response <- self.new(req.ctx, req.refId) case req := <-self.getChan: req.response <- self.get(req.user, req.id) + case req := <-self.listChan: + req.response <- self.list(req.user, req.password, req.trusted) case req := <-self.removeChan: req.response <- self.remove(req.user, req.id) } @@ -176,6 +216,7 @@ func (self *SessionStore) dispatchRequests() { type SessionStoreChan struct { newChan chan<- newSessionRequest getChan chan<- getSessionRequest + listChan chan listSessionsRequest removeChan chan<- removeSessionRequest } @@ -203,6 +244,19 @@ func (self *SessionStoreChan) Get(user, id string) (*SessionChan, string, int, s return res.session, res.refId, res.responsecode, res.errorstring } +func (self *SessionStoreChan) List(user, password string, trusted bool) (map[string]string, int, string) { + res_ch := make(chan listSessionsResponse) + req := listSessionsRequest{} + req.user = user + req.password = password + req.trusted = trusted + req.response = res_ch + self.listChan <- req + + res := <-res_ch + return res.sessions, res.responsecode, res.errorstring +} + func (self *SessionStoreChan) Remove(user, id string) (int, string) { res_ch := make(chan removeSessionResponse) req := removeSessionRequest{} @@ -219,6 +273,7 @@ func (self *SessionStore) GetInterface() *SessionStoreChan { ch := &SessionStoreChan{} ch.newChan = self.newChan ch.getChan = self.getChan + ch.listChan = self.listChan ch.removeChan = self.removeChan return ch } @@ -230,6 +285,7 @@ func (self *SessionStore) Cleanup() { close(self.done) close(self.newChan) close(self.getChan) + close(self.listChan) close(self.removeChan) } @@ -241,6 +297,7 @@ func NewSessionStore(conf *Config) (store *SessionStore, err error) { store.store = make(map[string]map[string]*SessionStoreElement) store.newChan = make(chan newSessionRequest, 10) store.getChan = make(chan getSessionRequest, 10) + store.listChan = make(chan listSessionsRequest, 10) store.removeChan = make(chan removeSessionRequest, 10) go store.dispatchRequests() -- cgit v0.10.2 From d4ef4d60994605e2a8251844c7eec166ee4094ec Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 30 Dec 2015 22:42:59 +0100 Subject: web socket handler is now completed diff --git a/session_store.go b/session_store.go index 0d149c6..9566293 100644 --- a/session_store.go +++ b/session_store.go @@ -89,6 +89,8 @@ type SessionStoreElement struct { type SessionStore struct { store map[string]map[string]*SessionStoreElement + conf *Config + rddb *RdDbChan quit chan bool done chan bool newChan chan newSessionRequest @@ -109,7 +111,7 @@ func (self *SessionStore) new(ctx *ImportContext, refId string) (resp newSession resp.responsecode = http.StatusOK resp.errorstring = "OK" if !ctx.Trusted { - if ok, err := ctx.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { + if ok, err := self.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { resp.responsecode = http.StatusInternalServerError resp.errorstring = err.Error() return @@ -127,6 +129,8 @@ func (self *SessionStore) new(ctx *ImportContext, refId string) (resp newSession if _, exists := self.store[ctx.UserName]; !exists { self.store[ctx.UserName] = make(map[string]*SessionStoreElement) } + ctx.conf = self.conf + ctx.rddb = self.rddb s := &SessionStoreElement{NewSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }), refId} self.store[ctx.UserName][resp.id] = s resp.session = self.store[ctx.UserName][resp.id].s.getInterface() @@ -151,18 +155,17 @@ func (self *SessionStore) get(user, id string) (resp getSessionResponse) { func (self *SessionStore) list(user, password string, trusted bool) (resp listSessionsResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" - // TODO: enable this check as soon as the session store handles the rddb itself - // if !trusted { - // if ok, err := self.rddb.CheckPassword(user, password); err != nil { - // resp.responsecode = http.StatusInternalServerError - // resp.errorstring = err.Error() - // return - // } else if !ok { - // resp.responsecode = http.StatusUnauthorized - // resp.errorstring = "invalid username and/or password" - // return - // } - // } + if !trusted { + if ok, err := self.rddb.CheckPassword(user, password); err != nil { + resp.responsecode = http.StatusInternalServerError + resp.errorstring = err.Error() + return + } else if !ok { + resp.responsecode = http.StatusUnauthorized + resp.errorstring = "invalid username and/or password" + return + } + } resp.sessions = make(map[string]string) if sessions, exists := self.store[user]; exists { for id, e := range sessions { @@ -289,9 +292,10 @@ func (self *SessionStore) Cleanup() { close(self.removeChan) } -func NewSessionStore(conf *Config) (store *SessionStore, err error) { +func NewSessionStore(conf *Config, rddb *RdDbChan) (store *SessionStore, err error) { store = new(SessionStore) - + store.conf = conf + store.rddb = rddb store.quit = make(chan bool) store.done = make(chan bool) store.store = make(map[string]map[string]*SessionStoreElement) -- cgit v0.10.2 From 399f26ca052c09ed54f812343a33d82560d9ae48 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 30 Dec 2015 23:44:30 +0100 Subject: improved sanity check for local files diff --git a/fetcher.go b/fetcher.go index 81072eb..f1bbb5e 100644 --- a/fetcher.go +++ b/fetcher.go @@ -158,7 +158,7 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData); !keep { + if keep := ctx.ProgressCallBack(1, "fetching", 0.0, ctx.ProgressCallBackData); !keep { ctx.ProgressCallBack = nil } } @@ -170,7 +170,23 @@ func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err err res.ErrorString = fmt.Sprintf("local-file open(): %s", err) return nil } + if info, err := src.Stat(); err != nil { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("local-file stat(): %s", err) + return nil + } else { + if info.IsDir() { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("'%s' is a directory", ctx.SourceFile) + return nil + } + } src.Close() + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } ctx.DeleteSourceFile = false ctx.DeleteSourceDir = false return -- cgit v0.10.2 From 774e626114dace5435f5f21dc9957d5da996b3bb Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 31 Dec 2015 10:46:40 +0100 Subject: improved coding style diff --git a/conf.go b/conf.go index 5f22782..d628563 100644 --- a/conf.go +++ b/conf.go @@ -39,15 +39,15 @@ type Config struct { configfile string RDXportEndpoint string TempDir string - db_host string - db_user string - db_passwd string - db_db string + dbHost string + dbUser string + dbPasswd string + dbDb string LocalFetchDir string ImportParamDefaults } -func get_ini_value(file ini.File, section string, key string, dflt string) string { +func getIniValue(file ini.File, section string, key string, dflt string) string { value, ok := file.Get(section, key) if ok { return value @@ -55,28 +55,28 @@ func get_ini_value(file ini.File, section string, key string, dflt string) strin return dflt } -func (self *Config) read_config_file() error { +func (self *Config) readConfigFile() error { file, err := ini.LoadFile(self.configfile) if err != nil { return err } - self.db_host = get_ini_value(file, "mySQL", "Hostname", "localhost") - self.db_user = get_ini_value(file, "mySQL", "Loginname", "rivendell") - self.db_passwd = get_ini_value(file, "mySQL", "Password", "letmein") - self.db_db = get_ini_value(file, "mySQL", "Database", "rivendell") + self.dbHost = getIniValue(file, "mySQL", "Hostname", "localhost") + self.dbUser = getIniValue(file, "mySQL", "Loginname", "rivendell") + self.dbPasswd = getIniValue(file, "mySQL", "Password", "letmein") + self.dbDb = getIniValue(file, "mySQL", "Database", "rivendell") return nil } -func NewConfig(configfile, rdxport_endpoint, temp_dir, local_fetch_dir *string) (conf *Config, err error) { +func NewConfig(configfile, rdxportEndpoint, tempDir, localFetchDir *string) (conf *Config, err error) { conf = new(Config) conf.configfile = *configfile - if err = conf.read_config_file(); err != nil { + if err = conf.readConfigFile(); err != nil { return } - conf.RDXportEndpoint = *rdxport_endpoint - conf.TempDir = *temp_dir - conf.LocalFetchDir = *local_fetch_dir + conf.RDXportEndpoint = *rdxportEndpoint + conf.TempDir = *tempDir + conf.LocalFetchDir = *localFetchDir conf.ImportParamDefaults.Channels = 2 conf.ImportParamDefaults.NormalizationLevel = -12 conf.ImportParamDefaults.AutotrimLevel = 0 diff --git a/core.go b/core.go index 3b8280a..1824db1 100644 --- a/core.go +++ b/core.go @@ -46,10 +46,10 @@ var ( func init() { curl.GlobalInit(curl.GLOBAL_ALL) - fetcher_init() + fetcherInit() } -type ImportProgressCB func(step int, step_name string, progress float64, userdata interface{}) bool +type ImportProgressCB func(step int, stepName string, progress float64, userdata interface{}) bool type ImportContext struct { conf *Config diff --git a/fetcher.go b/fetcher.go index f1bbb5e..55b814c 100644 --- a/fetcher.go +++ b/fetcher.go @@ -91,7 +91,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { return true } -func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { +func fetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) easy := curl.EasyInit() @@ -155,7 +155,7 @@ func FetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro return } -func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { +func fetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { if keep := ctx.ProgressCallBack(1, "fetching", 0.0, ctx.ProgressCallBackData); !keep { @@ -192,7 +192,7 @@ func FetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err err return } -func FetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { +func fetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { rhdl.Printf("Fake fetcher for '%s'", ctx.SourceUri) if duration, err := strconv.ParseUint(uri.Host, 10, 32); err != nil { @@ -234,28 +234,28 @@ type FetchFunc func(*ImportContext, *FetchResult, *url.URL) (err error) // home:// ????? var ( fetchers = map[string]FetchFunc{ - "local": FetchFileLocal, - "fake": FetchFileFake, + "local": fetchFileLocal, + "fake": fetchFileFake, } - curl_protos = map[string]bool{ + curlProtos = map[string]bool{ "http": false, "https": false, "ftp": false, "ftps": false, } ) -func fetcher_init() { +func fetcherInit() { info := curl.VersionInfo(curl.VERSION_FIRST) protos := info.Protocols for _, proto := range protos { - if _, ok := curl_protos[proto]; ok { + if _, ok := curlProtos[proto]; ok { rhdl.Printf("curl: enabling protocol %s", proto) - fetchers[proto] = FetchFileCurl - curl_protos[proto] = true + fetchers[proto] = fetchFileCurl + curlProtos[proto] = true } else { rhdl.Printf("curl: ignoring protocol %s", proto) } } - for proto, enabled := range curl_protos { + for proto, enabled := range curlProtos { if !enabled { rhl.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto) } diff --git a/importer.go b/importer.go index abc99e8..e4f979f 100644 --- a/importer.go +++ b/importer.go @@ -50,8 +50,8 @@ func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { } } -func add_cart(ctx *ImportContext, res *ImportResult) (err error) { - rhdl.Printf("importer: add_cart() called for cart: %d", ctx.Cart) +func addCart(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: addCart() called for cart: %d", ctx.Cart) if ctx.GroupName == "" { if err = ctx.getGroupOfCart(); err != nil { @@ -85,7 +85,7 @@ func add_cart(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -110,8 +110,8 @@ func add_cart(ctx *ImportContext, res *ImportResult) (err error) { return } -func add_cut(ctx *ImportContext, res *ImportResult) (err error) { - rhdl.Printf("importer: add_cut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) +func addCut(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: addCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -130,7 +130,7 @@ func add_cut(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -157,8 +157,8 @@ func add_cut(ctx *ImportContext, res *ImportResult) (err error) { return } -func remove_cart(ctx *ImportContext, res *ImportResult) (err error) { - rhdl.Printf("importer: remove_cart() called for cart: %d", ctx.Cart) +func removeCart(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: removeCart() called for cart: %d", ctx.Cart) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -177,7 +177,7 @@ func remove_cart(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -191,8 +191,8 @@ func remove_cart(ctx *ImportContext, res *ImportResult) (err error) { return } -func remove_cut(ctx *ImportContext, res *ImportResult) (err error) { - rhdl.Printf("importer: remove_cut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) +func removeCut(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: removeCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -214,7 +214,7 @@ func remove_cut(ctx *ImportContext, res *ImportResult) (err error) { w.Close() var resp *http.Response - if resp, err = send_post_request(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { return } defer resp.Body.Close() @@ -229,7 +229,7 @@ func remove_cut(ctx *ImportContext, res *ImportResult) (err error) { return } -func send_post_request(url string, b *bytes.Buffer, contenttype string) (resp *http.Response, err error) { +func sendPostRequest(url string, b *bytes.Buffer, contenttype string) (resp *http.Response, err error) { var req *http.Request if req, err = http.NewRequest("POST", url, b); err != nil { return @@ -245,7 +245,7 @@ func send_post_request(url string, b *bytes.Buffer, contenttype string) (resp *h return } -func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *curl.Form, err error) { +func importAudioCreateRequest(ctx *ImportContext, easy *curl.CURL) (form *curl.Form, err error) { form = curl.NewForm() if err = form.Add("COMMAND", "2"); err != nil { @@ -282,8 +282,8 @@ func import_audio_create_request(ctx *ImportContext, easy *curl.CURL) (form *cur return } -func import_audio(ctx *ImportContext, res *ImportResult) (err error) { - rhdl.Printf("importer: import_audio() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) +func importAudio(ctx *ImportContext, res *ImportResult) (err error) { + rhdl.Printf("importer: importAudio() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) easy := curl.EasyInit() if easy != nil { @@ -293,7 +293,7 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { easy.Setopt(curl.OPT_POST, true) var form *curl.Form - if form, err = import_audio_create_request(ctx, easy); err != nil { + if form, err = importAudioCreateRequest(ctx, easy); err != nil { return } easy.Setopt(curl.OPT_HTTPPOST, form) @@ -350,24 +350,24 @@ func import_audio(ctx *ImportContext, res *ImportResult) (err error) { return } -func add_cart_cut(ctx *ImportContext, res *ImportResult) (err error) { - if err = add_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { +func addCartCut(ctx *ImportContext, res *ImportResult) (err error) { + if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } - if err = add_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return remove_cart(ctx, &ImportResult{ResponseCode: http.StatusOK}) + if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return removeCart(ctx, &ImportResult{ResponseCode: http.StatusOK}) } return } -func remove_add_cart_cut(ctx *ImportContext, res *ImportResult) (err error) { - if err = remove_cart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { +func removeAddCartCut(ctx *ImportContext, res *ImportResult) (err error) { + if err = removeCart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { return } - return add_cart_cut(ctx, res) + return addCartCut(ctx, res) } -func is_cart_member_of_show(ctx *ImportContext, res *ImportResult, carts []uint) (found bool) { +func isCartMemberOfShow(ctx *ImportContext, res *ImportResult, carts []uint) (found bool) { if ctx.Cart == 0 { return true } @@ -382,33 +382,33 @@ func is_cart_member_of_show(ctx *ImportContext, res *ImportResult, carts []uint) return false } -func clear_show_carts(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { +func clearShowCarts(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { if ctx.ClearShowCarts { - orig_cart := ctx.Cart + origCart := ctx.Cart for _, cart := range carts { ctx.Cart = cart - if err = remove_cart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + if err = removeCart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { return } } - ctx.Cart = orig_cart + ctx.Cart = origCart } return } -func add_show_cart_cut(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { - if err = add_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { +func addShowCartCut(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { + if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } for _, cart := range carts { if cart == ctx.Cart { - if err = add_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return remove_cart(ctx, &ImportResult{ResponseCode: http.StatusOK}) + if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return removeCart(ctx, &ImportResult{ResponseCode: http.StatusOK}) } return } } - if err = remove_cart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = removeCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } res.ResponseCode = http.StatusForbidden @@ -416,7 +416,7 @@ func add_show_cart_cut(ctx *ImportContext, res *ImportResult, carts []uint) (err return } -func cleanup_files(ctx *ImportContext) { +func cleanupFiles(ctx *ImportContext) { if ctx.DeleteSourceFile { rhdl.Printf("importer: removing file: %s", ctx.SourceFile) if err := os.Remove(ctx.SourceFile); err != nil { @@ -435,7 +435,7 @@ func cleanup_files(ctx *ImportContext) { } func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { - defer cleanup_files(ctx) + defer cleanupFiles(ctx) rhl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) @@ -456,22 +456,22 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { rmCutOnErr := false res = &ImportResult{ResponseCode: http.StatusOK} if ctx.ShowId != 0 { // Import to a show - var show_carts []uint - if show_carts, err = ctx.getShowInfo(); err != nil { + var showCarts []uint + if showCarts, err = ctx.getShowInfo(); err != nil { return } - if !is_cart_member_of_show(ctx, res, show_carts) { + if !isCartMemberOfShow(ctx, res, showCarts) { return } - if err = clear_show_carts(ctx, res, show_carts); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + if err = clearShowCarts(ctx, res, showCarts); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { return } if ctx.ClearCart && !ctx.ClearShowCarts { - if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = removeAddCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } } else { - if err = add_show_cart_cut(ctx, res, show_carts); err != nil || res.ResponseCode != http.StatusOK { + if err = addShowCartCut(ctx, res, showCarts); err != nil || res.ResponseCode != http.StatusOK { return } } @@ -480,22 +480,22 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { if err = ctx.getMusicInfo(); err != nil { return } - if err = add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = addCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } rmCartOnErr = true } else if ctx.Cart != 0 && ctx.Cut == 0 { // Import to Cart if ctx.ClearCart { - if err = remove_add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = removeAddCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } rmCartOnErr = true } else { - if err = add_cut(ctx, res); err != nil { + if err = addCut(ctx, res); err != nil { return } if res.ResponseCode != http.StatusOK { - if err = add_cart_cut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = addCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } rmCartOnErr = true @@ -506,7 +506,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { } if ctx.Cart != 0 && ctx.Cut != 0 { // Import to specific Cut within Cart - if err = import_audio(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err = importAudio(ctx, res); err != nil || res.ResponseCode != http.StatusOK { if err != nil { rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", ctx.Cart, ctx.Cut, err) } else { @@ -515,11 +515,11 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { // Try to clean up after failed import rmres := ImportResult{ResponseCode: http.StatusOK} if rmCartOnErr { - if rerr := remove_cart(ctx, &rmres); rerr != nil { + if rerr := removeCart(ctx, &rmres); rerr != nil { return } } else if rmCutOnErr { - if rerr := remove_cut(ctx, &rmres); rerr != nil { + if rerr := removeCut(ctx, &rmres); rerr != nil { return } } diff --git a/rddb.go b/rddb.go index 8bec3c0..12bdba3 100644 --- a/rddb.go +++ b/rddb.go @@ -63,12 +63,12 @@ type getGroupOfCartRequest struct { } type getShowInfoResult struct { - title string - group string - carts []uint - norm_lvl int - trim_lvl int - err error + title string + group string + carts []uint + normLvl int + trimLvl int + err error } type getShowInfoRequest struct { @@ -87,9 +87,9 @@ type checkMusicGroupRequest struct { } type getMusicInfoResult struct { - norm_lvl int - trim_lvl int - err error + normLvl int + trimLvl int + err error } type getMusicInfoRequest struct { @@ -99,7 +99,7 @@ type getMusicInfoRequest struct { type RdDb struct { dbh *sql.DB - password_cache map[string]string + passwordCache map[string]string getPasswordChan chan getPasswordRequest getPasswordStmt *sql.Stmt getGroupOfCartChan chan getGroupOfCartRequest @@ -117,7 +117,7 @@ type RdDb struct { func (self *RdDb) init(conf *Config) (err error) { godrv.Register("SET CHARACTER SET utf8;") - dsn := fmt.Sprintf("tcp:%s:3306*%s/%s/%s", conf.db_host, conf.db_db, conf.db_user, conf.db_passwd) + dsn := fmt.Sprintf("tcp:%s:3306*%s/%s/%s", conf.dbHost, conf.dbDb, conf.dbUser, conf.dbPasswd) if self.dbh, err = sql.Open("mymysql", dsn); err != nil { return } @@ -153,7 +153,7 @@ func (self *RdDb) init(conf *Config) (err error) { func (self *RdDb) getPassword(username string, cached bool) (result getPasswordResult) { if cached { - result.password = self.password_cache[username] + result.password = self.passwordCache[username] } if result.password == "" { @@ -163,7 +163,7 @@ func (self *RdDb) getPassword(username string, cached bool) (result getPasswordR } return } - self.password_cache[username] = result.password + self.passwordCache[username] = result.password } return @@ -175,18 +175,18 @@ func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { return } defer rows.Close() - size_min := ^uint(0) + sizeMin := ^uint(0) for rows.Next() { var name string - var low_cart, high_cart uint - if result.err = rows.Scan(&name, &low_cart, &high_cart); result.err != nil { + var lowCart, highCart uint + if result.err = rows.Scan(&name, &lowCart, &highCart); result.err != nil { return } - if high_cart >= low_cart { - size := (high_cart - low_cart) + 1 - if size_min > size { + if highCart >= lowCart { + size := (highCart - lowCart) + 1 + if sizeMin > size { result.group = name - size_min = size + sizeMin = size } } } @@ -207,12 +207,12 @@ func (self *RdDb) getLogTableName(log string) (logtable string, err error) { return } -func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []uint, err error) { +func (self *RdDb) getShowCarts(log string, lowCart, highCart int) (carts []uint, err error) { var logtable string if logtable, err = self.getLogTableName(log); err != nil { return } - q := fmt.Sprintf("select CART_NUMBER from %s where CART_NUMBER >= %d and CART_NUMBER <= %d order by COUNT;", logtable, low_cart, high_cart) + q := fmt.Sprintf("select CART_NUMBER from %s where CART_NUMBER >= %d and CART_NUMBER <= %d order by COUNT;", logtable, lowCart, highCart) var rows *sql.Rows if rows, err = self.dbh.Query(q); err != nil { return @@ -231,17 +231,17 @@ func (self *RdDb) getShowCarts(log string, low_cart, high_cart int) (carts []uin func (self *RdDb) getShowInfo(showid uint) (result getShowInfoResult) { var macros string - var low_cart, high_cart int - result.err = self.getShowInfoStmt.QueryRow(showid).Scan(&result.title, ¯os, &result.group, &result.norm_lvl, &result.trim_lvl, &low_cart, &high_cart) + var lowCart, highCart int + result.err = self.getShowInfoStmt.QueryRow(showid).Scan(&result.title, ¯os, &result.group, &result.normLvl, &result.trimLvl, &lowCart, &highCart) if result.err != nil { if result.err == sql.ErrNoRows { result.err = fmt.Errorf("show '%d' not found", showid) } return } - result.norm_lvl /= 100 - result.trim_lvl /= 100 - result.carts, result.err = self.getShowCarts(showMacroRe.FindStringSubmatch(macros)[1], low_cart, high_cart) + result.normLvl /= 100 + result.trimLvl /= 100 + result.carts, result.err = self.getShowCarts(showMacroRe.FindStringSubmatch(macros)[1], lowCart, highCart) return } @@ -259,7 +259,7 @@ func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { } func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { - result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&result.norm_lvl, &result.trim_lvl) + result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&result.normLvl, &result.trimLvl) if result.err != nil { if result.err == sql.ErrNoRows { result.err = fmt.Errorf("music pool '%s' not found", group) @@ -301,14 +301,14 @@ type RdDbChan struct { } func (self *RdDbChan) GetPassword(username string, cached bool) (string, error) { - res_ch := make(chan getPasswordResult) + resCh := make(chan getPasswordResult) req := getPasswordRequest{} req.username = username req.cached = cached - req.response = res_ch + req.response = resCh self.getPasswordChan <- req - res := <-res_ch + res := <-resCh if res.err != nil { return "", res.err } @@ -319,14 +319,14 @@ func (self *RdDbChan) CheckPassword(username, password string) (bool, error) { cached := true for { - res_ch := make(chan getPasswordResult) + resCh := make(chan getPasswordResult) req := getPasswordRequest{} req.username = username req.cached = cached - req.response = res_ch + req.response = resCh self.getPasswordChan <- req - res := <-res_ch + res := <-resCh if res.err != nil { return false, res.err } @@ -343,13 +343,13 @@ func (self *RdDbChan) CheckPassword(username, password string) (bool, error) { } func (self *RdDbChan) GetGroupOfCart(cart uint) (string, error) { - res_ch := make(chan getGroupOfCartResult) + resCh := make(chan getGroupOfCartResult) req := getGroupOfCartRequest{} req.cart = cart - req.response = res_ch + req.response = resCh self.getGroupOfCartChan <- req - res := <-res_ch + res := <-resCh if res.err != nil { return "", res.err } @@ -357,27 +357,27 @@ func (self *RdDbChan) GetGroupOfCart(cart uint) (string, error) { } func (self *RdDbChan) GetShowInfo(showid uint) (string, int, int, []uint, error) { - res_ch := make(chan getShowInfoResult) + resCh := make(chan getShowInfoResult) req := getShowInfoRequest{} req.showid = showid - req.response = res_ch + req.response = resCh self.getShowInfoChan <- req - res := <-res_ch + res := <-resCh if res.err != nil { return "", 0, 0, nil, res.err } - return res.group, res.norm_lvl, res.trim_lvl, res.carts, nil + return res.group, res.normLvl, res.trimLvl, res.carts, nil } func (self *RdDbChan) CheckMusicGroup(groupname string) (bool, error) { - res_ch := make(chan checkMusicGroupResult) + resCh := make(chan checkMusicGroupResult) req := checkMusicGroupRequest{} req.group = groupname - req.response = res_ch + req.response = resCh self.checkMusicGroupChan <- req - res := <-res_ch + res := <-resCh if res.err != nil { return false, res.err } @@ -385,17 +385,17 @@ func (self *RdDbChan) CheckMusicGroup(groupname string) (bool, error) { } func (self *RdDbChan) GetMusicInfo(groupname string) (int, int, error) { - res_ch := make(chan getMusicInfoResult) + resCh := make(chan getMusicInfoResult) req := getMusicInfoRequest{} req.group = groupname - req.response = res_ch + req.response = resCh self.getMusicInfoChan <- req - res := <-res_ch + res := <-resCh if res.err != nil { return 0, 0, res.err } - return res.norm_lvl, res.trim_lvl, nil + return res.normLvl, res.trimLvl, nil } func (self *RdDb) GetInterface() *RdDbChan { @@ -439,7 +439,7 @@ func NewRdDb(conf *Config) (db *RdDb, err error) { db.quit = make(chan bool) db.done = make(chan bool) - db.password_cache = make(map[string]string) + db.passwordCache = make(map[string]string) db.getPasswordChan = make(chan getPasswordRequest, 10) db.getGroupOfCartChan = make(chan getGroupOfCartRequest, 10) db.getShowInfoChan = make(chan getShowInfoRequest, 10) diff --git a/session.go b/session.go index 36e7938..82ab3cf 100644 --- a/session.go +++ b/session.go @@ -92,13 +92,13 @@ type sessionAddDoneHandlerRequest struct { response chan<- sessionAddDoneHandlerResponse } -func session_progress_callback(step int, step_name string, progress float64, userdata interface{}) bool { +func sessionProgressCallback(step int, stepName string, progress float64, userdata interface{}) bool { out := userdata.(chan<- ProgressData) - out <- ProgressData{step, step_name, progress} + out <- ProgressData{step, stepName, progress} return true } -func session_import_run(ctx ImportContext, done chan<- ImportResult) { +func sessionImportRun(ctx ImportContext, done chan<- ImportResult) { if err := ctx.SanityCheck(); err != nil { done <- ImportResult{http.StatusBadRequest, err.Error(), 0, 0} return @@ -122,10 +122,10 @@ func session_import_run(ctx ImportContext, done chan<- ImportResult) { } func (self *Session) run(timeout time.Duration) { - self.ctx.ProgressCallBack = session_progress_callback + self.ctx.ProgressCallBack = sessionProgressCallback self.ctx.ProgressCallBackData = (chan<- ProgressData)(self.progressIntChan) self.ctx.Cancel = self.cancelIntChan - go session_import_run(self.ctx, self.doneIntChan) + go sessionImportRun(self.ctx, self.doneIntChan) self.state = SESSION_RUNNING self.timer.Reset(timeout) return @@ -247,37 +247,40 @@ func (self *SessionChan) Cancel() { } func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgressCB) error { - res_ch := make(chan sessionAddProgressHandlerResponse) + resCh := make(chan sessionAddProgressHandlerResponse) req := sessionAddProgressHandlerRequest{} req.userdata = userdata req.callback = cb - req.response = res_ch + req.response = resCh select { case self.addProgressChan <- req: default: return fmt.Errorf("session is about to be closed/removed") } - res := <-res_ch + res := <-resCh return res.err } func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) error { - res_ch := make(chan sessionAddDoneHandlerResponse) + resCh := make(chan sessionAddDoneHandlerResponse) req := sessionAddDoneHandlerRequest{} req.userdata = userdata req.callback = cb - req.response = res_ch + req.response = resCh select { case self.addDoneChan <- req: default: return fmt.Errorf("session is about to be closed/removed") } - res := <-res_ch + res := <-resCh return res.err } +// ********************************************************* +// Semi-Public Interface (only used by sessionStore) + func (self *Session) getInterface() *SessionChan { ch := &SessionChan{} ch.runChan = self.runChan @@ -287,7 +290,7 @@ func (self *Session) getInterface() *SessionChan { return ch } -func (self *Session) Cleanup() { +func (self *Session) cleanup() { self.quit <- true rhdl.Printf("waiting for session to close") <-self.done @@ -306,7 +309,7 @@ func (self *Session) Cleanup() { rhdl.Printf("session is now cleaned up") } -func NewSession(ctx *ImportContext, removeFunc func()) (session *Session) { +func newSession(ctx *ImportContext, removeFunc func()) (session *Session) { session = new(Session) session.state = SESSION_NEW session.removeFunc = removeFunc diff --git a/session_store.go b/session_store.go index 9566293..bb4043b 100644 --- a/session_store.go +++ b/session_store.go @@ -131,7 +131,7 @@ func (self *SessionStore) new(ctx *ImportContext, refId string) (resp newSession } ctx.conf = self.conf ctx.rddb = self.rddb - s := &SessionStoreElement{NewSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }), refId} + s := &SessionStoreElement{newSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }), refId} self.store[ctx.UserName][resp.id] = s resp.session = self.store[ctx.UserName][resp.id].s.getInterface() rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) @@ -179,7 +179,7 @@ func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" if session, exists := self.store[user][id]; exists { - go session.s.Cleanup() // cleanup could take a while -> don't block all the other stuff + go session.s.cleanup() // cleanup could take a while -> don't block all the other stuff delete(self.store[user], id) rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) if userstore, exists := self.store[user]; exists { @@ -224,51 +224,51 @@ type SessionStoreChan struct { } func (self *SessionStoreChan) New(ctx *ImportContext, refId string) (string, *SessionChan, int, string) { - res_ch := make(chan newSessionResponse) + resCh := make(chan newSessionResponse) req := newSessionRequest{} req.ctx = ctx req.refId = refId - req.response = res_ch + req.response = resCh self.newChan <- req - res := <-res_ch + res := <-resCh return res.id, res.session, res.responsecode, res.errorstring } func (self *SessionStoreChan) Get(user, id string) (*SessionChan, string, int, string) { - res_ch := make(chan getSessionResponse) + resCh := make(chan getSessionResponse) req := getSessionRequest{} req.user = user req.id = id - req.response = res_ch + req.response = resCh self.getChan <- req - res := <-res_ch + res := <-resCh return res.session, res.refId, res.responsecode, res.errorstring } func (self *SessionStoreChan) List(user, password string, trusted bool) (map[string]string, int, string) { - res_ch := make(chan listSessionsResponse) + resCh := make(chan listSessionsResponse) req := listSessionsRequest{} req.user = user req.password = password req.trusted = trusted - req.response = res_ch + req.response = resCh self.listChan <- req - res := <-res_ch + res := <-resCh return res.sessions, res.responsecode, res.errorstring } func (self *SessionStoreChan) Remove(user, id string) (int, string) { - res_ch := make(chan removeSessionResponse) + resCh := make(chan removeSessionResponse) req := removeSessionRequest{} req.user = user req.id = id - req.response = res_ch + req.response = resCh self.removeChan <- req - res := <-res_ch + res := <-resCh return res.responsecode, res.errorstring } -- cgit v0.10.2 From 1e2e49754634da258811d430a13f86f85a744e61 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 31 Dec 2015 14:18:50 +0100 Subject: all command line arguments can now be set using environment variables diff --git a/conf.go b/conf.go index d628563..59308a9 100644 --- a/conf.go +++ b/conf.go @@ -68,15 +68,15 @@ func (self *Config) readConfigFile() error { return nil } -func NewConfig(configfile, rdxportEndpoint, tempDir, localFetchDir *string) (conf *Config, err error) { +func NewConfig(configfile, rdxportEndpoint, tempDir, localFetchDir string) (conf *Config, err error) { conf = new(Config) - conf.configfile = *configfile + conf.configfile = configfile if err = conf.readConfigFile(); err != nil { return } - conf.RDXportEndpoint = *rdxportEndpoint - conf.TempDir = *tempDir - conf.LocalFetchDir = *localFetchDir + conf.RDXportEndpoint = rdxportEndpoint + conf.TempDir = tempDir + conf.LocalFetchDir = localFetchDir conf.ImportParamDefaults.Channels = 2 conf.ImportParamDefaults.NormalizationLevel = -12 conf.ImportParamDefaults.AutotrimLevel = 0 -- cgit v0.10.2 From 00a14226835ba540d2c7675d3fac7cc2ab7f7695 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Sun, 3 Jan 2016 20:15:47 +0100 Subject: make debug log runtime configurable diff --git a/core.go b/core.go index 1824db1..58633d8 100644 --- a/core.go +++ b/core.go @@ -25,9 +25,9 @@ package rhimport import ( - // "io/ioutil" "fmt" "github.com/golang-basic/go-curl" + "io/ioutil" "log" "os" ) @@ -40,11 +40,13 @@ const ( var ( bool2str = map[bool]string{false: "0", true: "1"} rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) - rhdl = log.New(os.Stderr, "[rhimport-dbg]\t", log.LstdFlags) - //rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) + rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) ) func init() { + if _, exists := os.LookupEnv("RHIMPORT_DEBUG"); exists { + rhdl.SetOutput(os.Stderr) + } curl.GlobalInit(curl.GLOBAL_ALL) fetcherInit() } -- cgit v0.10.2 From 6648578c53782af4334bb6e44b231d2cd7e22aa4 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 5 Jan 2016 15:24:49 +0100 Subject: minor refactoring diff --git a/core.go b/core.go index 58633d8..f74f422 100644 --- a/core.go +++ b/core.go @@ -52,6 +52,7 @@ func init() { } type ImportProgressCB func(step int, stepName string, progress float64, userdata interface{}) bool +type ImportDoneCB func(ImportResult, interface{}) bool type ImportContext struct { conf *Config diff --git a/session.go b/session.go index 82ab3cf..80acee1 100644 --- a/session.go +++ b/session.go @@ -62,7 +62,7 @@ type SessionProgressCB struct { } type SessionDoneCB struct { - cb func(ImportResult, interface{}) bool + cb ImportDoneCB userdata interface{} } @@ -88,7 +88,7 @@ type sessionAddDoneHandlerResponse struct { type sessionAddDoneHandlerRequest struct { userdata interface{} - callback func(ImportResult, interface{}) bool + callback ImportDoneCB response chan<- sessionAddDoneHandlerResponse } @@ -148,7 +148,7 @@ func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressC return } -func (self *Session) addDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) (resp sessionAddDoneHandlerResponse) { +func (self *Session) addDoneHandler(userdata interface{}, cb ImportDoneCB) (resp sessionAddDoneHandlerResponse) { if self.state != SESSION_NEW && self.state != SESSION_RUNNING { resp.err = fmt.Errorf("session is already done/canceled") } @@ -262,7 +262,7 @@ func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgr return res.err } -func (self *SessionChan) AddDoneHandler(userdata interface{}, cb func(ImportResult, interface{}) bool) error { +func (self *SessionChan) AddDoneHandler(userdata interface{}, cb ImportDoneCB) error { resCh := make(chan sessionAddDoneHandlerResponse) req := sessionAddDoneHandlerRequest{} req.userdata = userdata -- cgit v0.10.2 From 674db099c02dcbf39a336fa81b01deac3cc668ef Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 5 Jan 2016 15:33:38 +0100 Subject: replaces FetchResult by ImportResult diff --git a/core.go b/core.go index f74f422..bb92d30 100644 --- a/core.go +++ b/core.go @@ -54,6 +54,13 @@ func init() { type ImportProgressCB func(step int, stepName string, progress float64, userdata interface{}) bool type ImportDoneCB func(ImportResult, interface{}) bool +type ImportResult struct { + ResponseCode int + ErrorString string + Cart uint + Cut uint +} + type ImportContext struct { conf *Config rddb *RdDbChan diff --git a/fetcher.go b/fetcher.go index 55b814c..f5d5420 100644 --- a/fetcher.go +++ b/fetcher.go @@ -39,11 +39,6 @@ import ( "time" ) -type FetchResult struct { - ResponseCode int - ErrorString string -} - type FetcherCurlCBData struct { basepath string filename string @@ -91,7 +86,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { return true } -func fetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { +func fetchFileCurl(ctx *ImportContext, res *ImportResult, uri *url.URL) (err error) { rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) easy := curl.EasyInit() @@ -155,7 +150,7 @@ func fetchFileCurl(ctx *ImportContext, res *FetchResult, uri *url.URL) (err erro return } -func fetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err error) { +func fetchFileLocal(ctx *ImportContext, res *ImportResult, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { if keep := ctx.ProgressCallBack(1, "fetching", 0.0, ctx.ProgressCallBackData); !keep { @@ -192,7 +187,7 @@ func fetchFileLocal(ctx *ImportContext, res *FetchResult, uri *url.URL) (err err return } -func fetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { +func fetchFileFake(ctx *ImportContext, res *ImportResult, uri *url.URL) error { rhdl.Printf("Fake fetcher for '%s'", ctx.SourceUri) if duration, err := strconv.ParseUint(uri.Host, 10, 32); err != nil { @@ -226,7 +221,7 @@ func fetchFileFake(ctx *ImportContext, res *FetchResult, uri *url.URL) error { return nil } -type FetchFunc func(*ImportContext, *FetchResult, *url.URL) (err error) +type FetchFunc func(*ImportContext, *ImportResult, *url.URL) (err error) // TODO: implement fetchers for: // archiv:// @@ -262,7 +257,7 @@ func fetcherInit() { } } -func checkPassword(ctx *ImportContext, result *FetchResult) (err error) { +func checkPassword(ctx *ImportContext, result *ImportResult) (err error) { ok := false if ok, err = ctx.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { return @@ -274,8 +269,8 @@ func checkPassword(ctx *ImportContext, result *FetchResult) (err error) { return } -func FetchFile(ctx *ImportContext) (res *FetchResult, err error) { - res = &FetchResult{ResponseCode: http.StatusOK} +func FetchFile(ctx *ImportContext) (res *ImportResult, err error) { + res = &ImportResult{ResponseCode: http.StatusOK} var uri *url.URL if uri, err = url.Parse(ctx.SourceUri); err != nil { diff --git a/importer.go b/importer.go index e4f979f..4a0bece 100644 --- a/importer.go +++ b/importer.go @@ -35,13 +35,6 @@ import ( "path" ) -type ImportResult struct { - ResponseCode int - ErrorString string - Cart uint - Cut uint -} - func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { self.ResponseCode = rdres.ResponseCode self.ErrorString = rdres.ErrorString diff --git a/session.go b/session.go index 80acee1..e0f1a4b 100644 --- a/session.go +++ b/session.go @@ -108,7 +108,7 @@ func sessionImportRun(ctx ImportContext, done chan<- ImportResult) { done <- ImportResult{http.StatusInternalServerError, err.Error(), 0, 0} return } else if res.ResponseCode != http.StatusOK { - done <- ImportResult{res.ResponseCode, res.ErrorString, 0, 0} + done <- *res return } -- cgit v0.10.2 From 11dc798c4523bc83c9fb03c7ca74eb203e615ebe Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 5 Jan 2016 15:45:56 +0100 Subject: removed Import prefix of some types diff --git a/core.go b/core.go index bb92d30..9927be9 100644 --- a/core.go +++ b/core.go @@ -51,17 +51,17 @@ func init() { fetcherInit() } -type ImportProgressCB func(step int, stepName string, progress float64, userdata interface{}) bool -type ImportDoneCB func(ImportResult, interface{}) bool +type ProgressCB func(step int, stepName string, progress float64, userdata interface{}) bool +type DoneCB func(Result, interface{}) bool -type ImportResult struct { +type Result struct { ResponseCode int ErrorString string Cart uint Cut uint } -type ImportContext struct { +type Context struct { conf *Config rddb *RdDbChan UserName string @@ -81,13 +81,13 @@ type ImportContext struct { SourceFile string DeleteSourceFile bool DeleteSourceDir bool - ProgressCallBack ImportProgressCB + ProgressCallBack ProgressCB ProgressCallBackData interface{} Cancel <-chan bool } -func NewImportContext(conf *Config, rddb *RdDbChan, user string) *ImportContext { - ctx := new(ImportContext) +func NewContext(conf *Config, rddb *RdDbChan, user string) *Context { + ctx := new(Context) ctx.conf = conf ctx.rddb = rddb ctx.UserName = user @@ -112,7 +112,7 @@ func NewImportContext(conf *Config, rddb *RdDbChan, user string) *ImportContext return ctx } -func (ctx *ImportContext) SanityCheck() error { +func (ctx *Context) SanityCheck() error { if ctx.UserName == "" { return fmt.Errorf("empty Username is not allowed") } @@ -156,28 +156,28 @@ func (ctx *ImportContext) SanityCheck() error { return nil } -func (ctx *ImportContext) getPassword(cached bool) (err error) { +func (ctx *Context) getPassword(cached bool) (err error) { ctx.Password, err = ctx.rddb.GetPassword(ctx.UserName, cached) return } -func (ctx *ImportContext) getGroupOfCart() (err error) { +func (ctx *Context) getGroupOfCart() (err error) { ctx.GroupName, err = ctx.rddb.GetGroupOfCart(ctx.Cart) return } -func (ctx *ImportContext) getShowInfo() (carts []uint, err error) { +func (ctx *Context) getShowInfo() (carts []uint, err error) { ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.rddb.GetShowInfo(ctx.ShowId) ctx.Channels = 2 ctx.UseMetaData = true return } -func (ctx *ImportContext) checkMusicGroup() (bool, error) { +func (ctx *Context) checkMusicGroup() (bool, error) { return ctx.rddb.CheckMusicGroup(ctx.GroupName) } -func (ctx *ImportContext) getMusicInfo() (err error) { +func (ctx *Context) getMusicInfo() (err error) { ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.rddb.GetMusicInfo(ctx.GroupName) ctx.Channels = 2 ctx.UseMetaData = true diff --git a/fetcher.go b/fetcher.go index f5d5420..66c91a5 100644 --- a/fetcher.go +++ b/fetcher.go @@ -86,7 +86,7 @@ func curlWriteCallback(ptr []byte, userdata interface{}) bool { return true } -func fetchFileCurl(ctx *ImportContext, res *ImportResult, uri *url.URL) (err error) { +func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) easy := curl.EasyInit() @@ -150,7 +150,7 @@ func fetchFileCurl(ctx *ImportContext, res *ImportResult, uri *url.URL) (err err return } -func fetchFileLocal(ctx *ImportContext, res *ImportResult, uri *url.URL) (err error) { +func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) { rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) if ctx.ProgressCallBack != nil { if keep := ctx.ProgressCallBack(1, "fetching", 0.0, ctx.ProgressCallBackData); !keep { @@ -187,7 +187,7 @@ func fetchFileLocal(ctx *ImportContext, res *ImportResult, uri *url.URL) (err er return } -func fetchFileFake(ctx *ImportContext, res *ImportResult, uri *url.URL) error { +func fetchFileFake(ctx *Context, res *Result, uri *url.URL) error { rhdl.Printf("Fake fetcher for '%s'", ctx.SourceUri) if duration, err := strconv.ParseUint(uri.Host, 10, 32); err != nil { @@ -221,7 +221,7 @@ func fetchFileFake(ctx *ImportContext, res *ImportResult, uri *url.URL) error { return nil } -type FetchFunc func(*ImportContext, *ImportResult, *url.URL) (err error) +type FetchFunc func(*Context, *Result, *url.URL) (err error) // TODO: implement fetchers for: // archiv:// @@ -257,7 +257,7 @@ func fetcherInit() { } } -func checkPassword(ctx *ImportContext, result *ImportResult) (err error) { +func checkPassword(ctx *Context, result *Result) (err error) { ok := false if ok, err = ctx.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { return @@ -269,8 +269,8 @@ func checkPassword(ctx *ImportContext, result *ImportResult) (err error) { return } -func FetchFile(ctx *ImportContext) (res *ImportResult, err error) { - res = &ImportResult{ResponseCode: http.StatusOK} +func FetchFile(ctx *Context) (res *Result, err error) { + res = &Result{ResponseCode: http.StatusOK} var uri *url.URL if uri, err = url.Parse(ctx.SourceUri); err != nil { diff --git a/importer.go b/importer.go index 4a0bece..9af3765 100644 --- a/importer.go +++ b/importer.go @@ -35,7 +35,7 @@ import ( "path" ) -func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { +func (self *Result) fromRDWebResult(rdres *RDWebResult) { self.ResponseCode = rdres.ResponseCode self.ErrorString = rdres.ErrorString if rdres.AudioConvertError != 0 { @@ -43,7 +43,7 @@ func (self *ImportResult) fromRDWebResult(rdres *RDWebResult) { } } -func addCart(ctx *ImportContext, res *ImportResult) (err error) { +func addCart(ctx *Context, res *Result) (err error) { rhdl.Printf("importer: addCart() called for cart: %d", ctx.Cart) if ctx.GroupName == "" { @@ -103,7 +103,7 @@ func addCart(ctx *ImportContext, res *ImportResult) (err error) { return } -func addCut(ctx *ImportContext, res *ImportResult) (err error) { +func addCut(ctx *Context, res *Result) (err error) { rhdl.Printf("importer: addCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -150,7 +150,7 @@ func addCut(ctx *ImportContext, res *ImportResult) (err error) { return } -func removeCart(ctx *ImportContext, res *ImportResult) (err error) { +func removeCart(ctx *Context, res *Result) (err error) { rhdl.Printf("importer: removeCart() called for cart: %d", ctx.Cart) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -184,7 +184,7 @@ func removeCart(ctx *ImportContext, res *ImportResult) (err error) { return } -func removeCut(ctx *ImportContext, res *ImportResult) (err error) { +func removeCut(ctx *Context, res *Result) (err error) { rhdl.Printf("importer: removeCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) var b bytes.Buffer w := multipart.NewWriter(&b) @@ -238,7 +238,7 @@ func sendPostRequest(url string, b *bytes.Buffer, contenttype string) (resp *htt return } -func importAudioCreateRequest(ctx *ImportContext, easy *curl.CURL) (form *curl.Form, err error) { +func importAudioCreateRequest(ctx *Context, easy *curl.CURL) (form *curl.Form, err error) { form = curl.NewForm() if err = form.Add("COMMAND", "2"); err != nil { @@ -275,7 +275,7 @@ func importAudioCreateRequest(ctx *ImportContext, easy *curl.CURL) (form *curl.F return } -func importAudio(ctx *ImportContext, res *ImportResult) (err error) { +func importAudio(ctx *Context, res *Result) (err error) { rhdl.Printf("importer: importAudio() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) easy := curl.EasyInit() @@ -343,24 +343,24 @@ func importAudio(ctx *ImportContext, res *ImportResult) (err error) { return } -func addCartCut(ctx *ImportContext, res *ImportResult) (err error) { +func addCartCut(ctx *Context, res *Result) (err error) { if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return removeCart(ctx, &ImportResult{ResponseCode: http.StatusOK}) + return removeCart(ctx, &Result{ResponseCode: http.StatusOK}) } return } -func removeAddCartCut(ctx *ImportContext, res *ImportResult) (err error) { +func removeAddCartCut(ctx *Context, res *Result) (err error) { if err = removeCart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { return } return addCartCut(ctx, res) } -func isCartMemberOfShow(ctx *ImportContext, res *ImportResult, carts []uint) (found bool) { +func isCartMemberOfShow(ctx *Context, res *Result, carts []uint) (found bool) { if ctx.Cart == 0 { return true } @@ -375,7 +375,7 @@ func isCartMemberOfShow(ctx *ImportContext, res *ImportResult, carts []uint) (fo return false } -func clearShowCarts(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { +func clearShowCarts(ctx *Context, res *Result, carts []uint) (err error) { if ctx.ClearShowCarts { origCart := ctx.Cart for _, cart := range carts { @@ -389,14 +389,14 @@ func clearShowCarts(ctx *ImportContext, res *ImportResult, carts []uint) (err er return } -func addShowCartCut(ctx *ImportContext, res *ImportResult, carts []uint) (err error) { +func addShowCartCut(ctx *Context, res *Result, carts []uint) (err error) { if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { return } for _, cart := range carts { if cart == ctx.Cart { if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return removeCart(ctx, &ImportResult{ResponseCode: http.StatusOK}) + return removeCart(ctx, &Result{ResponseCode: http.StatusOK}) } return } @@ -409,7 +409,7 @@ func addShowCartCut(ctx *ImportContext, res *ImportResult, carts []uint) (err er return } -func cleanupFiles(ctx *ImportContext) { +func cleanupFiles(ctx *Context) { if ctx.DeleteSourceFile { rhdl.Printf("importer: removing file: %s", ctx.SourceFile) if err := os.Remove(ctx.SourceFile); err != nil { @@ -427,7 +427,7 @@ func cleanupFiles(ctx *ImportContext) { return } -func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { +func ImportFile(ctx *Context) (res *Result, err error) { defer cleanupFiles(ctx) rhl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) @@ -447,7 +447,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { rmCartOnErr := false rmCutOnErr := false - res = &ImportResult{ResponseCode: http.StatusOK} + res = &Result{ResponseCode: http.StatusOK} if ctx.ShowId != 0 { // Import to a show var showCarts []uint if showCarts, err = ctx.getShowInfo(); err != nil { @@ -506,7 +506,7 @@ func ImportFile(ctx *ImportContext) (res *ImportResult, err error) { rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) } // Try to clean up after failed import - rmres := ImportResult{ResponseCode: http.StatusOK} + rmres := Result{ResponseCode: http.StatusOK} if rmCartOnErr { if rerr := removeCart(ctx, &rmres); rerr != nil { return diff --git a/session.go b/session.go index e0f1a4b..8eab828 100644 --- a/session.go +++ b/session.go @@ -39,7 +39,7 @@ const ( ) type Session struct { - ctx ImportContext + ctx Context state int removeFunc func() done chan bool @@ -47,7 +47,7 @@ type Session struct { timer *time.Timer cancelIntChan chan bool progressIntChan chan ProgressData - doneIntChan chan ImportResult + doneIntChan chan Result runChan chan time.Duration cancelChan chan bool addProgressChan chan sessionAddProgressHandlerRequest @@ -57,12 +57,12 @@ type Session struct { } type SessionProgressCB struct { - cb ImportProgressCB + cb ProgressCB userdata interface{} } type SessionDoneCB struct { - cb ImportDoneCB + cb DoneCB userdata interface{} } @@ -78,7 +78,7 @@ type sessionAddProgressHandlerResponse struct { type sessionAddProgressHandlerRequest struct { userdata interface{} - callback ImportProgressCB + callback ProgressCB response chan<- sessionAddProgressHandlerResponse } @@ -88,7 +88,7 @@ type sessionAddDoneHandlerResponse struct { type sessionAddDoneHandlerRequest struct { userdata interface{} - callback ImportDoneCB + callback DoneCB response chan<- sessionAddDoneHandlerResponse } @@ -98,14 +98,14 @@ func sessionProgressCallback(step int, stepName string, progress float64, userda return true } -func sessionImportRun(ctx ImportContext, done chan<- ImportResult) { +func sessionRun(ctx Context, done chan<- Result) { if err := ctx.SanityCheck(); err != nil { - done <- ImportResult{http.StatusBadRequest, err.Error(), 0, 0} + done <- Result{http.StatusBadRequest, err.Error(), 0, 0} return } if res, err := FetchFile(&ctx); err != nil { - done <- ImportResult{http.StatusInternalServerError, err.Error(), 0, 0} + done <- Result{http.StatusInternalServerError, err.Error(), 0, 0} return } else if res.ResponseCode != http.StatusOK { done <- *res @@ -113,7 +113,7 @@ func sessionImportRun(ctx ImportContext, done chan<- ImportResult) { } if res, err := ImportFile(&ctx); err != nil { - done <- ImportResult{http.StatusInternalServerError, err.Error(), 0, 0} + done <- Result{http.StatusInternalServerError, err.Error(), 0, 0} return } else { done <- *res @@ -125,7 +125,7 @@ func (self *Session) run(timeout time.Duration) { self.ctx.ProgressCallBack = sessionProgressCallback self.ctx.ProgressCallBackData = (chan<- ProgressData)(self.progressIntChan) self.ctx.Cancel = self.cancelIntChan - go sessionImportRun(self.ctx, self.doneIntChan) + go sessionRun(self.ctx, self.doneIntChan) self.state = SESSION_RUNNING self.timer.Reset(timeout) return @@ -140,7 +140,7 @@ func (self *Session) cancel() { self.state = SESSION_CANCELED } -func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressCB) (resp sessionAddProgressHandlerResponse) { +func (self *Session) addProgressHandler(userdata interface{}, cb ProgressCB) (resp sessionAddProgressHandlerResponse) { if self.state != SESSION_NEW && self.state != SESSION_RUNNING { resp.err = fmt.Errorf("session is already done/canceled") } @@ -148,7 +148,7 @@ func (self *Session) addProgressHandler(userdata interface{}, cb ImportProgressC return } -func (self *Session) addDoneHandler(userdata interface{}, cb ImportDoneCB) (resp sessionAddDoneHandlerResponse) { +func (self *Session) addDoneHandler(userdata interface{}, cb DoneCB) (resp sessionAddDoneHandlerResponse) { if self.state != SESSION_NEW && self.state != SESSION_RUNNING { resp.err = fmt.Errorf("session is already done/canceled") } @@ -166,7 +166,7 @@ func (self *Session) callProgressHandler(p *ProgressData) { } } -func (self *Session) callDoneHandler(r *ImportResult) { +func (self *Session) callDoneHandler(r *Result) { for _, cb := range self.doneCBs { if cb.cb != nil { if keep := cb.cb(*r, cb.userdata); !keep { @@ -190,7 +190,7 @@ func (self *Session) dispatchRequests() { self.cancel() } self.state = SESSION_TIMEOUT - r := &ImportResult{500, "session timed out", 0, 0} + r := &Result{500, "session timed out", 0, 0} self.callDoneHandler(r) if self.removeFunc != nil { self.removeFunc() @@ -246,7 +246,7 @@ func (self *SessionChan) Cancel() { } } -func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgressCB) error { +func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ProgressCB) error { resCh := make(chan sessionAddProgressHandlerResponse) req := sessionAddProgressHandlerRequest{} req.userdata = userdata @@ -262,7 +262,7 @@ func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ImportProgr return res.err } -func (self *SessionChan) AddDoneHandler(userdata interface{}, cb ImportDoneCB) error { +func (self *SessionChan) AddDoneHandler(userdata interface{}, cb DoneCB) error { resCh := make(chan sessionAddDoneHandlerResponse) req := sessionAddDoneHandlerRequest{} req.userdata = userdata @@ -309,7 +309,7 @@ func (self *Session) cleanup() { rhdl.Printf("session is now cleaned up") } -func newSession(ctx *ImportContext, removeFunc func()) (session *Session) { +func newSession(ctx *Context, removeFunc func()) (session *Session) { session = new(Session) session.state = SESSION_NEW session.removeFunc = removeFunc @@ -318,7 +318,7 @@ func newSession(ctx *ImportContext, removeFunc func()) (session *Session) { session.timer = time.NewTimer(10 * time.Second) session.cancelIntChan = make(chan bool, 1) session.progressIntChan = make(chan ProgressData, 10) - session.doneIntChan = make(chan ImportResult, 1) + session.doneIntChan = make(chan Result, 1) session.runChan = make(chan time.Duration, 1) session.cancelChan = make(chan bool, 1) session.addProgressChan = make(chan sessionAddProgressHandlerRequest, 10) diff --git a/session_store.go b/session_store.go index bb4043b..db3f217 100644 --- a/session_store.go +++ b/session_store.go @@ -39,7 +39,7 @@ type newSessionResponse struct { } type newSessionRequest struct { - ctx *ImportContext + ctx *Context refId string response chan newSessionResponse } @@ -107,7 +107,7 @@ func generateSessionId() (string, error) { return base64.RawStdEncoding.EncodeToString(b[:]), nil } -func (self *SessionStore) new(ctx *ImportContext, refId string) (resp newSessionResponse) { +func (self *SessionStore) new(ctx *Context, refId string) (resp newSessionResponse) { resp.responsecode = http.StatusOK resp.errorstring = "OK" if !ctx.Trusted { @@ -223,7 +223,7 @@ type SessionStoreChan struct { removeChan chan<- removeSessionRequest } -func (self *SessionStoreChan) New(ctx *ImportContext, refId string) (string, *SessionChan, int, string) { +func (self *SessionStoreChan) New(ctx *Context, refId string) (string, *SessionChan, int, string) { resCh := make(chan newSessionResponse) req := newSessionRequest{} req.ctx = ctx -- cgit v0.10.2 From 489d4cc9c6a84728ac18e6b65b45b933f34eb5e4 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 5 Jan 2016 15:59:12 +0100 Subject: streamlined the usage of rhimport.NewContext diff --git a/core.go b/core.go index 9927be9..986327b 100644 --- a/core.go +++ b/core.go @@ -86,11 +86,11 @@ type Context struct { Cancel <-chan bool } -func NewContext(conf *Config, rddb *RdDbChan, user string) *Context { +func NewContext(conf *Config, rddb *RdDbChan) *Context { ctx := new(Context) ctx.conf = conf ctx.rddb = rddb - ctx.UserName = user + ctx.UserName = "" ctx.Password = "" ctx.Trusted = false ctx.ShowId = 0 -- cgit v0.10.2 From f2eeb80a1f3dc0e3626c889b13a74e87f7daae1c Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 5 Jan 2016 16:45:26 +0100 Subject: updated copyright header diff --git a/conf.go b/conf.go index 59308a9..2c91ed9 100644 --- a/conf.go +++ b/conf.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // diff --git a/core.go b/core.go index 986327b..29e8e28 100644 --- a/core.go +++ b/core.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // diff --git a/fetcher.go b/fetcher.go index 66c91a5..4c6eec3 100644 --- a/fetcher.go +++ b/fetcher.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // diff --git a/importer.go b/importer.go index 9af3765..8a38958 100644 --- a/importer.go +++ b/importer.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // diff --git a/rddb.go b/rddb.go index 12bdba3..4c9c6c8 100644 --- a/rddb.go +++ b/rddb.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // diff --git a/rdxport_responses.go b/rdxport_responses.go index 392f58f..2871408 100644 --- a/rdxport_responses.go +++ b/rdxport_responses.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // diff --git a/session.go b/session.go index 8eab828..66705ec 100644 --- a/session.go +++ b/session.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // diff --git a/session_store.go b/session_store.go index db3f217..8b48c0a 100644 --- a/session_store.go +++ b/session_store.go @@ -4,7 +4,7 @@ // The Radio Helsinki Rivendell Import Daemon // // -// Copyright (C) 2015 Christian Pointner +// Copyright (C) 2015-2016 Christian Pointner // // This file is part of rhimportd. // -- cgit v0.10.2 From 6741874cf6746eecdef94da3bc34e73ac139d363 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Tue, 5 Jan 2016 22:48:58 +0100 Subject: switched to more update2date go-curl lib diff --git a/core.go b/core.go index 29e8e28..b5c7784 100644 --- a/core.go +++ b/core.go @@ -26,7 +26,7 @@ package rhimport import ( "fmt" - "github.com/golang-basic/go-curl" + "github.com/andelf/go-curl" "io/ioutil" "log" "os" diff --git a/fetcher.go b/fetcher.go index 4c6eec3..7a196e4 100644 --- a/fetcher.go +++ b/fetcher.go @@ -26,7 +26,7 @@ package rhimport import ( "fmt" - "github.com/golang-basic/go-curl" + "github.com/andelf/go-curl" "io/ioutil" "mime" "net/http" diff --git a/importer.go b/importer.go index 8a38958..0702db4 100644 --- a/importer.go +++ b/importer.go @@ -28,7 +28,7 @@ import ( "bufio" "bytes" "fmt" - "github.com/golang-basic/go-curl" + "github.com/andelf/go-curl" "mime/multipart" "net/http" "os" -- cgit v0.10.2 From 2e548139a6806bab10d1b184e0120dbcb1f99c1f Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 8 Jan 2016 01:51:37 +0100 Subject: move rddb to rhrd-go diff --git a/conf.go b/conf.go index 2c91ed9..b7e8f88 100644 --- a/conf.go +++ b/conf.go @@ -24,10 +24,6 @@ package rhimport -import ( - "github.com/vaughan0/go-ini" -) - type ImportParamDefaults struct { Channels uint NormalizationLevel int @@ -36,44 +32,14 @@ type ImportParamDefaults struct { } type Config struct { - configfile string RDXportEndpoint string TempDir string - dbHost string - dbUser string - dbPasswd string - dbDb string LocalFetchDir string ImportParamDefaults } -func getIniValue(file ini.File, section string, key string, dflt string) string { - value, ok := file.Get(section, key) - if ok { - return value - } - return dflt -} - -func (self *Config) readConfigFile() error { - file, err := ini.LoadFile(self.configfile) - if err != nil { - return err - } - - self.dbHost = getIniValue(file, "mySQL", "Hostname", "localhost") - self.dbUser = getIniValue(file, "mySQL", "Loginname", "rivendell") - self.dbPasswd = getIniValue(file, "mySQL", "Password", "letmein") - self.dbDb = getIniValue(file, "mySQL", "Database", "rivendell") - return nil -} - -func NewConfig(configfile, rdxportEndpoint, tempDir, localFetchDir string) (conf *Config, err error) { +func NewConfig(rdxportEndpoint, tempDir, localFetchDir string) (conf *Config) { conf = new(Config) - conf.configfile = configfile - if err = conf.readConfigFile(); err != nil { - return - } conf.RDXportEndpoint = rdxportEndpoint conf.TempDir = tempDir conf.LocalFetchDir = localFetchDir diff --git a/core.go b/core.go index b5c7784..8b6f982 100644 --- a/core.go +++ b/core.go @@ -27,6 +27,7 @@ package rhimport import ( "fmt" "github.com/andelf/go-curl" + "helsinki.at/rhrd-go/rddb" "io/ioutil" "log" "os" @@ -63,7 +64,7 @@ type Result struct { type Context struct { conf *Config - rddb *RdDbChan + db *rddb.DBChan UserName string Password string Trusted bool @@ -86,10 +87,10 @@ type Context struct { Cancel <-chan bool } -func NewContext(conf *Config, rddb *RdDbChan) *Context { +func NewContext(conf *Config, db *rddb.DBChan) *Context { ctx := new(Context) ctx.conf = conf - ctx.rddb = rddb + ctx.db = db ctx.UserName = "" ctx.Password = "" ctx.Trusted = false @@ -157,28 +158,28 @@ func (ctx *Context) SanityCheck() error { } func (ctx *Context) getPassword(cached bool) (err error) { - ctx.Password, err = ctx.rddb.GetPassword(ctx.UserName, cached) + ctx.Password, err = ctx.db.GetPassword(ctx.UserName, cached) return } func (ctx *Context) getGroupOfCart() (err error) { - ctx.GroupName, err = ctx.rddb.GetGroupOfCart(ctx.Cart) + ctx.GroupName, err = ctx.db.GetGroupOfCart(ctx.Cart) return } func (ctx *Context) getShowInfo() (carts []uint, err error) { - ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.rddb.GetShowInfo(ctx.ShowId) + ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.db.GetShowInfo(ctx.ShowId) ctx.Channels = 2 ctx.UseMetaData = true return } func (ctx *Context) checkMusicGroup() (bool, error) { - return ctx.rddb.CheckMusicGroup(ctx.GroupName) + return ctx.db.CheckMusicGroup(ctx.GroupName) } func (ctx *Context) getMusicInfo() (err error) { - ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.rddb.GetMusicInfo(ctx.GroupName) + ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.db.GetMusicInfo(ctx.GroupName) ctx.Channels = 2 ctx.UseMetaData = true ctx.Cart = 0 diff --git a/fetcher.go b/fetcher.go index 7a196e4..e5fa29f 100644 --- a/fetcher.go +++ b/fetcher.go @@ -259,7 +259,7 @@ func fetcherInit() { func checkPassword(ctx *Context, result *Result) (err error) { ok := false - if ok, err = ctx.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { + if ok, err = ctx.db.CheckPassword(ctx.UserName, ctx.Password); err != nil { return } if !ok { diff --git a/rddb.go b/rddb.go deleted file mode 100644 index 4c9c6c8..0000000 --- a/rddb.go +++ /dev/null @@ -1,455 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "database/sql" - "fmt" - "github.com/ziutek/mymysql/godrv" - "regexp" - "strings" -) - -var ( - showMacroRe = regexp.MustCompile(`^LL 1 ([^ ]+) 0\!$`) - mysqlTableNameRe = regexp.MustCompile(`^[_0-9a-zA-Z-]+$`) -) - -const ( - DB_VERSION = 245 -) - -type getPasswordResult struct { - password string - err error -} - -type getPasswordRequest struct { - username string - cached bool - response chan<- getPasswordResult -} - -type getGroupOfCartResult struct { - group string - err error -} - -type getGroupOfCartRequest struct { - cart uint - response chan<- getGroupOfCartResult -} - -type getShowInfoResult struct { - title string - group string - carts []uint - normLvl int - trimLvl int - err error -} - -type getShowInfoRequest struct { - showid uint - response chan<- getShowInfoResult -} - -type checkMusicGroupResult struct { - ismusic bool - err error -} - -type checkMusicGroupRequest struct { - group string - response chan<- checkMusicGroupResult -} - -type getMusicInfoResult struct { - normLvl int - trimLvl int - err error -} - -type getMusicInfoRequest struct { - group string - response chan<- getMusicInfoResult -} - -type RdDb struct { - dbh *sql.DB - passwordCache map[string]string - getPasswordChan chan getPasswordRequest - getPasswordStmt *sql.Stmt - getGroupOfCartChan chan getGroupOfCartRequest - getGroupOfCartStmt *sql.Stmt - getShowInfoChan chan getShowInfoRequest - getShowInfoStmt *sql.Stmt - checkMusicGroupChan chan checkMusicGroupRequest - checkMusicGroupStmt *sql.Stmt - getMusicInfoChan chan getMusicInfoRequest - getMusicInfoStmt *sql.Stmt - quit chan bool - done chan bool -} - -func (self *RdDb) init(conf *Config) (err error) { - godrv.Register("SET CHARACTER SET utf8;") - - dsn := fmt.Sprintf("tcp:%s:3306*%s/%s/%s", conf.dbHost, conf.dbDb, conf.dbUser, conf.dbPasswd) - if self.dbh, err = sql.Open("mymysql", dsn); err != nil { - return - } - - var dbver int - err = self.dbh.QueryRow("select DB from VERSION;").Scan(&dbver) - if err != nil { - err = fmt.Errorf("fetching version: %s", err) - return - } - if dbver != DB_VERSION { - err = fmt.Errorf("version mismatch is %d, should be %d", dbver, DB_VERSION) - return - } - - if self.getPasswordStmt, err = self.dbh.Prepare("select PASSWORD from USERS where LOGIN_NAME = ?;"); err != nil { - return - } - if self.getGroupOfCartStmt, err = self.dbh.Prepare("select NAME,DEFAULT_LOW_CART,DEFAULT_HIGH_CART from GROUPS where DEFAULT_LOW_CART <= ? and DEFAULT_HIGH_CART >= ?;"); err != nil { - return - } - if self.getShowInfoStmt, err = self.dbh.Prepare("select CART.TITLE,CART.MACROS,DROPBOXES.GROUP_NAME,DROPBOXES.NORMALIZATION_LEVEL,DROPBOXES.AUTOTRIM_LEVEL,GROUPS.DEFAULT_LOW_CART,GROUPS.DEFAULT_HIGH_CART from CART, DROPBOXES, GROUPS where CART.NUMBER = DROPBOXES.TO_CART and GROUPS.NAME = DROPBOXES.GROUP_NAME and CART.NUMBER = ?;"); err != nil { - return - } - if self.checkMusicGroupStmt, err = self.dbh.Prepare("select count(*) from DROPBOXES where GROUP_NAME = ? and SET_USER_DEFINED like \"M;%\";"); err != nil { - return - } - if self.getMusicInfoStmt, err = self.dbh.Prepare("select NORMALIZATION_LEVEL,AUTOTRIM_LEVEL from DROPBOXES where DROPBOXES.GROUP_NAME = ?;"); err != nil { - return - } - return -} - -func (self *RdDb) getPassword(username string, cached bool) (result getPasswordResult) { - if cached { - result.password = self.passwordCache[username] - } - - if result.password == "" { - if result.err = self.getPasswordStmt.QueryRow(username).Scan(&result.password); result.err != nil { - if result.err == sql.ErrNoRows { - result.err = fmt.Errorf("user '%s' not known by rivendell", username) - } - return - } - self.passwordCache[username] = result.password - } - - return -} - -func (self *RdDb) getGroupOfCart(cart uint) (result getGroupOfCartResult) { - var rows *sql.Rows - if rows, result.err = self.getGroupOfCartStmt.Query(cart, cart); result.err != nil { - return - } - defer rows.Close() - sizeMin := ^uint(0) - for rows.Next() { - var name string - var lowCart, highCart uint - if result.err = rows.Scan(&name, &lowCart, &highCart); result.err != nil { - return - } - if highCart >= lowCart { - size := (highCart - lowCart) + 1 - if sizeMin > size { - result.group = name - sizeMin = size - } - } - } - if result.err = rows.Err(); result.err != nil { - return - } - if result.group == "" { - result.err = fmt.Errorf("cart is outside of all group cart ranges") - } - return -} - -func (self *RdDb) getLogTableName(log string) (logtable string, err error) { - logtable = strings.Replace(log, " ", "_", -1) + "_LOG" - if !mysqlTableNameRe.MatchString(logtable) { - return "", fmt.Errorf("the log table name contains illegal charecters: %s", logtable) - } - return -} - -func (self *RdDb) getShowCarts(log string, lowCart, highCart int) (carts []uint, err error) { - var logtable string - if logtable, err = self.getLogTableName(log); err != nil { - return - } - q := fmt.Sprintf("select CART_NUMBER from %s where CART_NUMBER >= %d and CART_NUMBER <= %d order by COUNT;", logtable, lowCart, highCart) - var rows *sql.Rows - if rows, err = self.dbh.Query(q); err != nil { - return - } - defer rows.Close() - for rows.Next() { - var cart uint - if err = rows.Scan(&cart); err != nil { - return - } - carts = append(carts, cart) - } - err = rows.Err() - return -} - -func (self *RdDb) getShowInfo(showid uint) (result getShowInfoResult) { - var macros string - var lowCart, highCart int - result.err = self.getShowInfoStmt.QueryRow(showid).Scan(&result.title, ¯os, &result.group, &result.normLvl, &result.trimLvl, &lowCart, &highCart) - if result.err != nil { - if result.err == sql.ErrNoRows { - result.err = fmt.Errorf("show '%d' not found", showid) - } - return - } - result.normLvl /= 100 - result.trimLvl /= 100 - result.carts, result.err = self.getShowCarts(showMacroRe.FindStringSubmatch(macros)[1], lowCart, highCart) - return -} - -func (self *RdDb) checkMusicGroup(group string) (result checkMusicGroupResult) { - var cnt int - if result.err = self.checkMusicGroupStmt.QueryRow(group).Scan(&cnt); result.err != nil { - if result.err == sql.ErrNoRows { - result.err = nil - result.ismusic = false - } - return - } - result.ismusic = cnt > 0 - return -} - -func (self *RdDb) getMusicInfo(group string) (result getMusicInfoResult) { - result.err = self.getMusicInfoStmt.QueryRow(group).Scan(&result.normLvl, &result.trimLvl) - if result.err != nil { - if result.err == sql.ErrNoRows { - result.err = fmt.Errorf("music pool '%s' not found", group) - } - return - } - return -} - -func (self *RdDb) dispatchRequests() { - defer func() { self.done <- true }() - for { - select { - case <-self.quit: - return - case req := <-self.getPasswordChan: - req.response <- self.getPassword(req.username, req.cached) - case req := <-self.getGroupOfCartChan: - req.response <- self.getGroupOfCart(req.cart) - case req := <-self.getShowInfoChan: - req.response <- self.getShowInfo(req.showid) - case req := <-self.checkMusicGroupChan: - req.response <- self.checkMusicGroup(req.group) - case req := <-self.getMusicInfoChan: - req.response <- self.getMusicInfo(req.group) - } - } -} - -// ********************************************************* -// Public Interface - -type RdDbChan struct { - getPasswordChan chan<- getPasswordRequest - getGroupOfCartChan chan<- getGroupOfCartRequest - getShowInfoChan chan<- getShowInfoRequest - checkMusicGroupChan chan<- checkMusicGroupRequest - getMusicInfoChan chan<- getMusicInfoRequest -} - -func (self *RdDbChan) GetPassword(username string, cached bool) (string, error) { - resCh := make(chan getPasswordResult) - req := getPasswordRequest{} - req.username = username - req.cached = cached - req.response = resCh - self.getPasswordChan <- req - - res := <-resCh - if res.err != nil { - return "", res.err - } - return res.password, nil -} - -func (self *RdDbChan) CheckPassword(username, password string) (bool, error) { - cached := true - - for { - resCh := make(chan getPasswordResult) - req := getPasswordRequest{} - req.username = username - req.cached = cached - req.response = resCh - self.getPasswordChan <- req - - res := <-resCh - if res.err != nil { - return false, res.err - } - if password == res.password { - return true, nil - } - if cached { - cached = false - } else { - break - } - } - return false, nil -} - -func (self *RdDbChan) GetGroupOfCart(cart uint) (string, error) { - resCh := make(chan getGroupOfCartResult) - req := getGroupOfCartRequest{} - req.cart = cart - req.response = resCh - self.getGroupOfCartChan <- req - - res := <-resCh - if res.err != nil { - return "", res.err - } - return res.group, nil -} - -func (self *RdDbChan) GetShowInfo(showid uint) (string, int, int, []uint, error) { - resCh := make(chan getShowInfoResult) - req := getShowInfoRequest{} - req.showid = showid - req.response = resCh - self.getShowInfoChan <- req - - res := <-resCh - if res.err != nil { - return "", 0, 0, nil, res.err - } - return res.group, res.normLvl, res.trimLvl, res.carts, nil -} - -func (self *RdDbChan) CheckMusicGroup(groupname string) (bool, error) { - resCh := make(chan checkMusicGroupResult) - req := checkMusicGroupRequest{} - req.group = groupname - req.response = resCh - self.checkMusicGroupChan <- req - - res := <-resCh - if res.err != nil { - return false, res.err - } - return res.ismusic, nil -} - -func (self *RdDbChan) GetMusicInfo(groupname string) (int, int, error) { - resCh := make(chan getMusicInfoResult) - req := getMusicInfoRequest{} - req.group = groupname - req.response = resCh - self.getMusicInfoChan <- req - - res := <-resCh - if res.err != nil { - return 0, 0, res.err - } - return res.normLvl, res.trimLvl, nil -} - -func (self *RdDb) GetInterface() *RdDbChan { - ch := &RdDbChan{} - ch.getPasswordChan = self.getPasswordChan - ch.getGroupOfCartChan = self.getGroupOfCartChan - ch.getShowInfoChan = self.getShowInfoChan - ch.checkMusicGroupChan = self.checkMusicGroupChan - ch.getMusicInfoChan = self.getMusicInfoChan - return ch -} - -func (self *RdDb) Cleanup() { - self.quit <- true - <-self.done - close(self.quit) - close(self.done) - close(self.getPasswordChan) - if self.dbh != nil { - self.dbh.Close() - } - if self.getPasswordStmt != nil { - self.getPasswordStmt.Close() - } - if self.getGroupOfCartStmt != nil { - self.getGroupOfCartStmt.Close() - } - if self.getShowInfoStmt != nil { - self.getShowInfoStmt.Close() - } - if self.checkMusicGroupStmt != nil { - self.checkMusicGroupStmt.Close() - } - if self.getMusicInfoStmt != nil { - self.getMusicInfoStmt.Close() - } -} - -func NewRdDb(conf *Config) (db *RdDb, err error) { - db = new(RdDb) - - db.quit = make(chan bool) - db.done = make(chan bool) - db.passwordCache = make(map[string]string) - db.getPasswordChan = make(chan getPasswordRequest, 10) - db.getGroupOfCartChan = make(chan getGroupOfCartRequest, 10) - db.getShowInfoChan = make(chan getShowInfoRequest, 10) - db.checkMusicGroupChan = make(chan checkMusicGroupRequest, 10) - db.getMusicInfoChan = make(chan getMusicInfoRequest, 10) - - if err = db.init(conf); err != nil { - return - } - - go db.dispatchRequests() - return -} diff --git a/session_store.go b/session_store.go index 8b48c0a..e47366e 100644 --- a/session_store.go +++ b/session_store.go @@ -28,6 +28,7 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "helsinki.at/rhrd-go/rddb" "net/http" ) @@ -90,7 +91,7 @@ type SessionStoreElement struct { type SessionStore struct { store map[string]map[string]*SessionStoreElement conf *Config - rddb *RdDbChan + db *rddb.DBChan quit chan bool done chan bool newChan chan newSessionRequest @@ -111,7 +112,7 @@ func (self *SessionStore) new(ctx *Context, refId string) (resp newSessionRespon resp.responsecode = http.StatusOK resp.errorstring = "OK" if !ctx.Trusted { - if ok, err := self.rddb.CheckPassword(ctx.UserName, ctx.Password); err != nil { + if ok, err := self.db.CheckPassword(ctx.UserName, ctx.Password); err != nil { resp.responsecode = http.StatusInternalServerError resp.errorstring = err.Error() return @@ -130,7 +131,7 @@ func (self *SessionStore) new(ctx *Context, refId string) (resp newSessionRespon self.store[ctx.UserName] = make(map[string]*SessionStoreElement) } ctx.conf = self.conf - ctx.rddb = self.rddb + ctx.db = self.db s := &SessionStoreElement{newSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }), refId} self.store[ctx.UserName][resp.id] = s resp.session = self.store[ctx.UserName][resp.id].s.getInterface() @@ -156,7 +157,7 @@ func (self *SessionStore) list(user, password string, trusted bool) (resp listSe resp.responsecode = http.StatusOK resp.errorstring = "OK" if !trusted { - if ok, err := self.rddb.CheckPassword(user, password); err != nil { + if ok, err := self.db.CheckPassword(user, password); err != nil { resp.responsecode = http.StatusInternalServerError resp.errorstring = err.Error() return @@ -292,10 +293,10 @@ func (self *SessionStore) Cleanup() { close(self.removeChan) } -func NewSessionStore(conf *Config, rddb *RdDbChan) (store *SessionStore, err error) { +func NewSessionStore(conf *Config, db *rddb.DBChan) (store *SessionStore, err error) { store = new(SessionStore) store.conf = conf - store.rddb = rddb + store.db = db store.quit = make(chan bool) store.done = make(chan bool) store.store = make(map[string]map[string]*SessionStoreElement) -- cgit v0.10.2 From 9cd0b1783c0c90c68c4840b5d317e9135e07774e Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 8 Jan 2016 01:55:14 +0100 Subject: moved all db handling directly to context diff --git a/core.go b/core.go index 8b6f982..98acd35 100644 --- a/core.go +++ b/core.go @@ -162,6 +162,10 @@ func (ctx *Context) getPassword(cached bool) (err error) { return } +func (ctx *Context) CheckPassword() (bool, error) { + return ctx.db.CheckPassword(ctx.UserName, ctx.Password) +} + func (ctx *Context) getGroupOfCart() (err error) { ctx.GroupName, err = ctx.db.GetGroupOfCart(ctx.Cart) return diff --git a/fetcher.go b/fetcher.go index e5fa29f..2d99be4 100644 --- a/fetcher.go +++ b/fetcher.go @@ -259,7 +259,7 @@ func fetcherInit() { func checkPassword(ctx *Context, result *Result) (err error) { ok := false - if ok, err = ctx.db.CheckPassword(ctx.UserName, ctx.Password); err != nil { + if ok, err = ctx.CheckPassword(); err != nil { return } if !ok { -- cgit v0.10.2 From e88ef9d360843541bd348b35099dda1b15c6c896 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Fri, 8 Jan 2016 02:06:30 +0100 Subject: prepare export rhimport package to rhrd-go repo diff --git a/conf.go b/conf.go deleted file mode 100644 index b7e8f88..0000000 --- a/conf.go +++ /dev/null @@ -1,51 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -type ImportParamDefaults struct { - Channels uint - NormalizationLevel int - AutotrimLevel int - UseMetaData bool -} - -type Config struct { - RDXportEndpoint string - TempDir string - LocalFetchDir string - ImportParamDefaults -} - -func NewConfig(rdxportEndpoint, tempDir, localFetchDir string) (conf *Config) { - conf = new(Config) - conf.RDXportEndpoint = rdxportEndpoint - conf.TempDir = tempDir - conf.LocalFetchDir = localFetchDir - conf.ImportParamDefaults.Channels = 2 - conf.ImportParamDefaults.NormalizationLevel = -12 - conf.ImportParamDefaults.AutotrimLevel = 0 - conf.ImportParamDefaults.UseMetaData = true - return -} diff --git a/core.go b/core.go deleted file mode 100644 index 98acd35..0000000 --- a/core.go +++ /dev/null @@ -1,192 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "fmt" - "github.com/andelf/go-curl" - "helsinki.at/rhrd-go/rddb" - "io/ioutil" - "log" - "os" -) - -const ( - CART_MAX = 999999 - CUT_MAX = 999 -) - -var ( - bool2str = map[bool]string{false: "0", true: "1"} - rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) - rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) -) - -func init() { - if _, exists := os.LookupEnv("RHIMPORT_DEBUG"); exists { - rhdl.SetOutput(os.Stderr) - } - curl.GlobalInit(curl.GLOBAL_ALL) - fetcherInit() -} - -type ProgressCB func(step int, stepName string, progress float64, userdata interface{}) bool -type DoneCB func(Result, interface{}) bool - -type Result struct { - ResponseCode int - ErrorString string - Cart uint - Cut uint -} - -type Context struct { - conf *Config - db *rddb.DBChan - UserName string - Password string - Trusted bool - ShowId uint - ClearShowCarts bool - GroupName string - Cart uint - ClearCart bool - Cut uint - Channels uint - NormalizationLevel int - AutotrimLevel int - UseMetaData bool - SourceUri string - SourceFile string - DeleteSourceFile bool - DeleteSourceDir bool - ProgressCallBack ProgressCB - ProgressCallBackData interface{} - Cancel <-chan bool -} - -func NewContext(conf *Config, db *rddb.DBChan) *Context { - ctx := new(Context) - ctx.conf = conf - ctx.db = db - ctx.UserName = "" - ctx.Password = "" - ctx.Trusted = false - ctx.ShowId = 0 - ctx.ClearShowCarts = false - ctx.GroupName = "" - ctx.Cart = 0 - ctx.ClearCart = false - ctx.Cut = 0 - ctx.Channels = conf.ImportParamDefaults.Channels - ctx.NormalizationLevel = conf.ImportParamDefaults.NormalizationLevel - ctx.AutotrimLevel = conf.ImportParamDefaults.AutotrimLevel - ctx.UseMetaData = conf.ImportParamDefaults.UseMetaData - ctx.SourceFile = "" - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - ctx.ProgressCallBack = nil - ctx.Cancel = nil - - return ctx -} - -func (ctx *Context) SanityCheck() error { - if ctx.UserName == "" { - return fmt.Errorf("empty Username is not allowed") - } - if ctx.Password == "" && !ctx.Trusted { - return fmt.Errorf("empty Password on untrusted control interface is not allowed") - } - if ctx.ShowId != 0 { - if ctx.ShowId != 0 && ctx.ShowId > CART_MAX { - return fmt.Errorf("ShowId %d is outside of allowed range (0 < show-id < %d)", ctx.ShowId, CART_MAX) - } - if ctx.Cart != 0 && ctx.Cart > CART_MAX { - return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) - } - return nil - } - if ctx.GroupName != "" { - ismusic, err := ctx.checkMusicGroup() - if err != nil { - return err - } - if !ismusic { - return fmt.Errorf("supplied GroupName '%s' is not a music pool", ctx.GroupName) - } - if ctx.Cart != 0 || ctx.Cut != 0 { - return fmt.Errorf("Cart and Cut must not be supplied when importing into a music group") - } - return nil - } - if ctx.Cart == 0 { - return fmt.Errorf("either ShowId, PoolName or CartNumber must be supplied") - } - if ctx.Cart > CART_MAX { - return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) - } - if ctx.Cut != 0 && ctx.Cut > CUT_MAX { - return fmt.Errorf("Cut %d is outside of allowed range (0 < cart < %d)", ctx.Cut, CUT_MAX) - } - if ctx.Channels != 1 && ctx.Channels != 2 { - return fmt.Errorf("channles must be 1 or 2") - } - return nil -} - -func (ctx *Context) getPassword(cached bool) (err error) { - ctx.Password, err = ctx.db.GetPassword(ctx.UserName, cached) - return -} - -func (ctx *Context) CheckPassword() (bool, error) { - return ctx.db.CheckPassword(ctx.UserName, ctx.Password) -} - -func (ctx *Context) getGroupOfCart() (err error) { - ctx.GroupName, err = ctx.db.GetGroupOfCart(ctx.Cart) - return -} - -func (ctx *Context) getShowInfo() (carts []uint, err error) { - ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.db.GetShowInfo(ctx.ShowId) - ctx.Channels = 2 - ctx.UseMetaData = true - return -} - -func (ctx *Context) checkMusicGroup() (bool, error) { - return ctx.db.CheckMusicGroup(ctx.GroupName) -} - -func (ctx *Context) getMusicInfo() (err error) { - ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.db.GetMusicInfo(ctx.GroupName) - ctx.Channels = 2 - ctx.UseMetaData = true - ctx.Cart = 0 - ctx.Cut = 0 - return -} diff --git a/fetcher.go b/fetcher.go deleted file mode 100644 index 2d99be4..0000000 --- a/fetcher.go +++ /dev/null @@ -1,294 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "fmt" - "github.com/andelf/go-curl" - "io/ioutil" - "mime" - "net/http" - "net/url" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "time" -) - -type FetcherCurlCBData struct { - basepath string - filename string - remotename string - *os.File -} - -func (self *FetcherCurlCBData) Cleanup() { - if self.File != nil { - self.File.Close() - } -} - -func curlHeaderCallback(ptr []byte, userdata interface{}) bool { - hdr := fmt.Sprintf("%s", ptr) - data := userdata.(*FetcherCurlCBData) - - if strings.HasPrefix(hdr, "Content-Disposition:") { - if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil { - if mediatype == "attachment" { - data.filename = data.basepath + "/" + params["filename"] - } - } - } - return true -} - -func curlWriteCallback(ptr []byte, userdata interface{}) bool { - data := userdata.(*FetcherCurlCBData) - if data.File == nil { - if data.filename == "" { - data.filename = data.basepath + "/" + data.remotename - } - fp, err := os.OpenFile(data.filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - rhl.Printf("Unable to create file %s: %s", data.filename, err) - return false - } - data.File = fp - } - if _, err := data.File.Write(ptr); err != nil { - rhl.Printf("Unable to write file %s: %s", data.filename, err) - return false - } - return true -} - -func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { - rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) - - easy := curl.EasyInit() - if easy != nil { - defer easy.Cleanup() - - easy.Setopt(curl.OPT_FOLLOWLOCATION, true) - easy.Setopt(curl.OPT_URL, ctx.SourceUri) - - cbdata := &FetcherCurlCBData{remotename: path.Base(uri.Path)} - defer cbdata.Cleanup() - if cbdata.basepath, err = ioutil.TempDir(ctx.conf.TempDir, "rhimportd-"); err != nil { - return - } - - easy.Setopt(curl.OPT_HEADERFUNCTION, curlHeaderCallback) - easy.Setopt(curl.OPT_HEADERDATA, cbdata) - - easy.Setopt(curl.OPT_WRITEFUNCTION, curlWriteCallback) - easy.Setopt(curl.OPT_WRITEDATA, cbdata) - - easy.Setopt(curl.OPT_NOPROGRESS, false) - easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { - if ctx.Cancel != nil && len(ctx.Cancel) > 0 { - rhl.Printf("downloading '%s' got canceled", ctx.SourceUri) - res.ResponseCode = http.StatusNoContent - res.ErrorString = "canceled" - return false - } - - if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(1, "downloading", dlnow/dltotal, ctx.ProgressCallBackData); !keep { - ctx.ProgressCallBack = nil - } - } - return true - }) - easy.Setopt(curl.OPT_PROGRESSDATA, ctx) - - if err = easy.Perform(); err != nil { - if cbdata.File != nil { - rhdl.Printf("Removing stale file: %s", cbdata.filename) - os.Remove(cbdata.filename) - os.Remove(path.Dir(cbdata.filename)) - } - if res.ResponseCode == http.StatusNoContent { - err = nil - } else { - err = fmt.Errorf("curl-fetcher('%s'): %s", ctx.SourceUri, err) - } - return - } - - ctx.SourceFile = cbdata.filename - ctx.DeleteSourceFile = true - ctx.DeleteSourceDir = true - } else { - err = fmt.Errorf("Error initializing libcurl") - } - - return -} - -func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) { - rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) - if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(1, "fetching", 0.0, ctx.ProgressCallBackData); !keep { - ctx.ProgressCallBack = nil - } - } - - ctx.SourceFile = filepath.Join(ctx.conf.LocalFetchDir, path.Clean("/"+uri.Path)) - var src *os.File - if src, err = os.Open(ctx.SourceFile); err != nil { - res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("local-file open(): %s", err) - return nil - } - if info, err := src.Stat(); err != nil { - res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("local-file stat(): %s", err) - return nil - } else { - if info.IsDir() { - res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("'%s' is a directory", ctx.SourceFile) - return nil - } - } - src.Close() - if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData); !keep { - ctx.ProgressCallBack = nil - } - } - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - return -} - -func fetchFileFake(ctx *Context, res *Result, uri *url.URL) error { - rhdl.Printf("Fake fetcher for '%s'", ctx.SourceUri) - - if duration, err := strconv.ParseUint(uri.Host, 10, 32); err != nil { - err = nil - res.ResponseCode = http.StatusBadRequest - res.ErrorString = "invalid duration (must be a positive integer)" - } else { - for i := uint(0); i < uint(duration); i++ { - if ctx.Cancel != nil && len(ctx.Cancel) > 0 { - rhl.Printf("faking got canceled") - res.ResponseCode = http.StatusNoContent - res.ErrorString = "canceled" - return nil - } - if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(1, "faking", float64(i)/float64(duration), ctx.ProgressCallBackData); !keep { - ctx.ProgressCallBack = nil - } - } - time.Sleep(100 * time.Millisecond) - } - if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(1, "faking", 1.0, ctx.ProgressCallBackData); !keep { - ctx.ProgressCallBack = nil - } - } - ctx.SourceFile = "/nonexistend/fake.mp3" - ctx.DeleteSourceFile = false - ctx.DeleteSourceDir = false - } - return nil -} - -type FetchFunc func(*Context, *Result, *url.URL) (err error) - -// TODO: implement fetchers for: -// archiv:// -// public:// -// home:// ????? -var ( - fetchers = map[string]FetchFunc{ - "local": fetchFileLocal, - "fake": fetchFileFake, - } - curlProtos = map[string]bool{ - "http": false, "https": false, - "ftp": false, "ftps": false, - } -) - -func fetcherInit() { - info := curl.VersionInfo(curl.VERSION_FIRST) - protos := info.Protocols - for _, proto := range protos { - if _, ok := curlProtos[proto]; ok { - rhdl.Printf("curl: enabling protocol %s", proto) - fetchers[proto] = fetchFileCurl - curlProtos[proto] = true - } else { - rhdl.Printf("curl: ignoring protocol %s", proto) - } - } - for proto, enabled := range curlProtos { - if !enabled { - rhl.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto) - } - } -} - -func checkPassword(ctx *Context, result *Result) (err error) { - ok := false - if ok, err = ctx.CheckPassword(); err != nil { - return - } - if !ok { - result.ResponseCode = http.StatusUnauthorized - result.ErrorString = "invalid username and/or password" - } - return -} - -func FetchFile(ctx *Context) (res *Result, err error) { - res = &Result{ResponseCode: http.StatusOK} - - var uri *url.URL - if uri, err = url.Parse(ctx.SourceUri); err != nil { - res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("parsing uri: %s", err) - return res, nil - } - - if !ctx.Trusted { - if err = checkPassword(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - } - - if fetcher, ok := fetchers[uri.Scheme]; ok { - err = fetcher(ctx, res, uri) - } else { - err = fmt.Errorf("No fetcher for uri scheme '%s' found.", uri.Scheme) - } - return -} diff --git a/importer.go b/importer.go deleted file mode 100644 index 0702db4..0000000 --- a/importer.go +++ /dev/null @@ -1,528 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "bufio" - "bytes" - "fmt" - "github.com/andelf/go-curl" - "mime/multipart" - "net/http" - "os" - "path" -) - -func (self *Result) fromRDWebResult(rdres *RDWebResult) { - self.ResponseCode = rdres.ResponseCode - self.ErrorString = rdres.ErrorString - if rdres.AudioConvertError != 0 { - self.ErrorString += fmt.Sprintf(", Audio Convert Error: %d", rdres.AudioConvertError) - } -} - -func addCart(ctx *Context, res *Result) (err error) { - rhdl.Printf("importer: addCart() called for cart: %d", ctx.Cart) - - if ctx.GroupName == "" { - if err = ctx.getGroupOfCart(); err != nil { - return - } - } - - var b bytes.Buffer - w := multipart.NewWriter(&b) - - if err = w.WriteField("COMMAND", "12"); err != nil { - return - } - if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { - return - } - if err = w.WriteField("PASSWORD", ctx.Password); err != nil { - return - } - if err = w.WriteField("GROUP_NAME", ctx.GroupName); err != nil { - return - } - if err = w.WriteField("TYPE", "audio"); err != nil { - return - } - if ctx.Cart != 0 { - if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { - return - } - } - w.Close() - - var resp *http.Response - if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { - return - } - res.fromRDWebResult(rdres) - res.Cart = ctx.Cart - return - } - var cartadd *RDCartAdd - if cartadd, err = NewRDCartAddFromXML(resp.Body); err != nil { - return - } - res.ResponseCode = resp.StatusCode - res.ErrorString = "OK" - res.Cart = cartadd.Carts[0].Number - ctx.Cart = res.Cart - return -} - -func addCut(ctx *Context, res *Result) (err error) { - rhdl.Printf("importer: addCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) - var b bytes.Buffer - w := multipart.NewWriter(&b) - - if err = w.WriteField("COMMAND", "10"); err != nil { - return - } - if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { - return - } - if err = w.WriteField("PASSWORD", ctx.Password); err != nil { - return - } - if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { - return - } - w.Close() - - var resp *http.Response - if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { - return - } - res.fromRDWebResult(rdres) - res.Cart = ctx.Cart - res.Cut = ctx.Cut - return - } - var cutadd *RDCutAdd - if cutadd, err = NewRDCutAddFromXML(resp.Body); err != nil { - return - } - res.ResponseCode = resp.StatusCode - res.ErrorString = "OK" - res.Cart = ctx.Cart - res.Cut = cutadd.Cuts[0].Number - ctx.Cut = cutadd.Cuts[0].Number - return -} - -func removeCart(ctx *Context, res *Result) (err error) { - rhdl.Printf("importer: removeCart() called for cart: %d", ctx.Cart) - var b bytes.Buffer - w := multipart.NewWriter(&b) - - if err = w.WriteField("COMMAND", "13"); err != nil { - return - } - if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { - return - } - if err = w.WriteField("PASSWORD", ctx.Password); err != nil { - return - } - if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { - return - } - w.Close() - - var resp *http.Response - if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { - return - } - defer resp.Body.Close() - - var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { - return - } - res.fromRDWebResult(rdres) - res.Cart = ctx.Cart - return -} - -func removeCut(ctx *Context, res *Result) (err error) { - rhdl.Printf("importer: removeCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) - var b bytes.Buffer - w := multipart.NewWriter(&b) - - if err = w.WriteField("COMMAND", "11"); err != nil { - return - } - if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { - return - } - if err = w.WriteField("PASSWORD", ctx.Password); err != nil { - return - } - if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { - return - } - if err = w.WriteField("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { - return - } - w.Close() - - var resp *http.Response - if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { - return - } - defer resp.Body.Close() - - var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { - return - } - res.fromRDWebResult(rdres) - res.Cart = ctx.Cart - res.Cut = ctx.Cut - return -} - -func sendPostRequest(url string, b *bytes.Buffer, contenttype string) (resp *http.Response, err error) { - var req *http.Request - if req, err = http.NewRequest("POST", url, b); err != nil { - return - } - if contenttype != "" { - req.Header.Set("Content-Type", contenttype) - } - - client := &http.Client{} - if resp, err = client.Do(req); err != nil { - return - } - return -} - -func importAudioCreateRequest(ctx *Context, easy *curl.CURL) (form *curl.Form, err error) { - form = curl.NewForm() - - if err = form.Add("COMMAND", "2"); err != nil { - return - } - if err = form.Add("LOGIN_NAME", ctx.UserName); err != nil { - return - } - if err = form.Add("PASSWORD", ctx.Password); err != nil { - return - } - if err = form.Add("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { - return - } - if err = form.Add("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { - return - } - if err = form.Add("CHANNELS", fmt.Sprintf("%d", ctx.Channels)); err != nil { - return - } - if err = form.Add("NORMALIZATION_LEVEL", fmt.Sprintf("%d", ctx.NormalizationLevel)); err != nil { - return - } - if err = form.Add("AUTOTRIM_LEVEL", fmt.Sprintf("%d", ctx.AutotrimLevel)); err != nil { - return - } - if err = form.Add("USE_METADATA", bool2str[ctx.UseMetaData]); err != nil { - return - } - if err = form.AddFile("FILENAME", ctx.SourceFile); err != nil { - return - } - - return -} - -func importAudio(ctx *Context, res *Result) (err error) { - rhdl.Printf("importer: importAudio() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) - easy := curl.EasyInit() - - if easy != nil { - defer easy.Cleanup() - - easy.Setopt(curl.OPT_URL, ctx.conf.RDXportEndpoint) - easy.Setopt(curl.OPT_POST, true) - - var form *curl.Form - if form, err = importAudioCreateRequest(ctx, easy); err != nil { - return - } - easy.Setopt(curl.OPT_HTTPPOST, form) - easy.Setopt(curl.OPT_HTTPHEADER, []string{"Expect:"}) - - var resbody bytes.Buffer - easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool { - b := userdata.(*bytes.Buffer) - b.Write(ptr) - return true - }) - easy.Setopt(curl.OPT_WRITEDATA, &resbody) - - easy.Setopt(curl.OPT_NOPROGRESS, false) - easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { - if ctx.Cancel != nil && len(ctx.Cancel) > 0 { - res.ResponseCode = http.StatusNoContent - res.ErrorString = "canceled" - return false - } - - if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(2, "importing", ulnow/ultotal, ctx.ProgressCallBackData); !keep { - ctx.ProgressCallBack = nil - } - } - return true - }) - easy.Setopt(curl.OPT_PROGRESSDATA, ctx) - - if err = easy.Perform(); err != nil { - if res.ResponseCode == http.StatusNoContent { - rhl.Printf("import to cart/cat %d/%d got canceled", ctx.Cart, ctx.Cut) - res.Cart = ctx.Cart - res.Cut = ctx.Cut - err = nil - } else { - err = fmt.Errorf("importer: %s", err) - } - return - } - - var rdres *RDWebResult - if rdres, err = NewRDWebResultFromXML(bufio.NewReader(&resbody)); err != nil { - return - } - res.fromRDWebResult(rdres) - res.Cart = ctx.Cart - res.Cut = ctx.Cut - } else { - err = fmt.Errorf("Error initializing libcurl") - } - - return -} - -func addCartCut(ctx *Context, res *Result) (err error) { - if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return removeCart(ctx, &Result{ResponseCode: http.StatusOK}) - } - return -} - -func removeAddCartCut(ctx *Context, res *Result) (err error) { - if err = removeCart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { - return - } - return addCartCut(ctx, res) -} - -func isCartMemberOfShow(ctx *Context, res *Result, carts []uint) (found bool) { - if ctx.Cart == 0 { - return true - } - for _, cart := range carts { - if cart == ctx.Cart { - return true - } - } - res.ResponseCode = http.StatusBadRequest - res.ErrorString = fmt.Sprintf("Requested cart %d is not a member of show: %d", ctx.Cart, ctx.ShowId) - res.Cart = ctx.Cart - return false -} - -func clearShowCarts(ctx *Context, res *Result, carts []uint) (err error) { - if ctx.ClearShowCarts { - origCart := ctx.Cart - for _, cart := range carts { - ctx.Cart = cart - if err = removeCart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { - return - } - } - ctx.Cart = origCart - } - return -} - -func addShowCartCut(ctx *Context, res *Result, carts []uint) (err error) { - if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - for _, cart := range carts { - if cart == ctx.Cart { - if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return removeCart(ctx, &Result{ResponseCode: http.StatusOK}) - } - return - } - } - if err = removeCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - res.ResponseCode = http.StatusForbidden - res.ErrorString = fmt.Sprintf("Show %d has no free carts left", ctx.ShowId) - return -} - -func cleanupFiles(ctx *Context) { - if ctx.DeleteSourceFile { - rhdl.Printf("importer: removing file: %s", ctx.SourceFile) - if err := os.Remove(ctx.SourceFile); err != nil { - rhl.Printf("importer: error removing source file: %s", err) - return - } - if ctx.DeleteSourceDir { - dir := path.Dir(ctx.SourceFile) - rhdl.Printf("importer: also removing directory: %s", dir) - if err := os.Remove(dir); err != nil { - rhl.Printf("importer: error removing source directory: %s", err) - } - } - } - return -} - -func ImportFile(ctx *Context) (res *Result, err error) { - defer cleanupFiles(ctx) - - rhl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) - - if ctx.ProgressCallBack != nil { - if keep := ctx.ProgressCallBack(2, "importing", 0.0, ctx.ProgressCallBackData); !keep { - ctx.ProgressCallBack = nil - } - } - - // TODO: on trusted interfaces we should call getPassword again with cached=false after 401's - if ctx.Trusted { - if err = ctx.getPassword(true); err != nil { - return - } - } - - rmCartOnErr := false - rmCutOnErr := false - res = &Result{ResponseCode: http.StatusOK} - if ctx.ShowId != 0 { // Import to a show - var showCarts []uint - if showCarts, err = ctx.getShowInfo(); err != nil { - return - } - if !isCartMemberOfShow(ctx, res, showCarts) { - return - } - if err = clearShowCarts(ctx, res, showCarts); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { - return - } - if ctx.ClearCart && !ctx.ClearShowCarts { - if err = removeAddCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - } else { - if err = addShowCartCut(ctx, res, showCarts); err != nil || res.ResponseCode != http.StatusOK { - return - } - } - rmCartOnErr = true - } else if ctx.GroupName != "" { // Import to music pool - if err = ctx.getMusicInfo(); err != nil { - return - } - if err = addCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - rmCartOnErr = true - } else if ctx.Cart != 0 && ctx.Cut == 0 { // Import to Cart - if ctx.ClearCart { - if err = removeAddCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - rmCartOnErr = true - } else { - if err = addCut(ctx, res); err != nil { - return - } - if res.ResponseCode != http.StatusOK { - if err = addCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - return - } - rmCartOnErr = true - } else { - rmCutOnErr = true - } - } - } - - if ctx.Cart != 0 && ctx.Cut != 0 { // Import to specific Cut within Cart - if err = importAudio(ctx, res); err != nil || res.ResponseCode != http.StatusOK { - if err != nil { - rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", ctx.Cart, ctx.Cut, err) - } else { - rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) - } - // Try to clean up after failed import - rmres := Result{ResponseCode: http.StatusOK} - if rmCartOnErr { - if rerr := removeCart(ctx, &rmres); rerr != nil { - return - } - } else if rmCutOnErr { - if rerr := removeCut(ctx, &rmres); rerr != nil { - return - } - } - } else { - rhl.Printf("File got succesfully imported into Cart/Cut %d/%d", res.Cart, res.Cut) - } - } else { - res.ResponseCode = http.StatusBadRequest - res.ErrorString = "importer: The request doesn't contain enough information to be processed" - } - - return -} diff --git a/rdxport_responses.go b/rdxport_responses.go deleted file mode 100644 index 2871408..0000000 --- a/rdxport_responses.go +++ /dev/null @@ -1,150 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "encoding/xml" - "fmt" - "io" -) - -type RDWebResult struct { - ResponseCode int `xml:"ResponseCode"` - ErrorString string `xml:"ErrorString"` - AudioConvertError int `xml:"AudioConvertError"` -} - -func NewRDWebResultFromXML(data io.Reader) (res *RDWebResult, err error) { - decoder := xml.NewDecoder(data) - res = &RDWebResult{} - if xmlerr := decoder.Decode(res); xmlerr != nil { - err = fmt.Errorf("Error parsing XML response: %s", xmlerr) - return - } - return -} - -type RDCartAdd struct { - Carts []RDCart `xml:"cart"` -} - -type RDCart struct { - Number uint `xml:"number"` - Type string `xml:"type"` - GroupName string `xml:"groupName"` - Title string `xml:"title"` - Artist string `xml:"artist"` - Album string `xml:"album"` - Year string `xml:"year"` - Label string `xml:"label"` - Client string `xml:"client"` - Agency string `xml:"agency"` - Publisher string `xml:"publisher"` - Composer string `xml:"composer"` - UserDefined string `xml:"userDefined"` - UsageCode uint `xml:"usageCode"` - ForcedLength string `xml:"forcedLength"` - AverageLength string `xml:"averageLength"` - LengthDeviation string `xml:"lengthDeviation"` - AverageSegueLength string `xml:"averageSegueLenth"` - AverageHookLength string `xml:"averageHookLength"` - CutQuantity uint `xml:"cutQuantity"` - LastCutPlayed uint `xml:"lastCutPlayed"` - Validity uint `xml:"validity"` - EnforceLength bool `xml:"enforceLength"` - Asynchronous bool `xml:"asyncronous"` - Owner string `xml:"owner"` - MetadataDatetime string `xml:"metadataDatetime"` -} - -func NewRDCartAddFromXML(data io.Reader) (cart *RDCartAdd, err error) { - decoder := xml.NewDecoder(data) - cart = &RDCartAdd{} - if xmlerr := decoder.Decode(cart); xmlerr != nil { - err = fmt.Errorf("Error parsing XML response: %s", xmlerr) - return - } - return -} - -type RDCutAdd struct { - Cuts []RDCut `xml:"cut"` -} - -type RDCut struct { - Name string `xml:"cutName"` - CartNumber uint `xml:"cartNumber"` - Number uint `xml:"cutNumber"` - EverGreen bool `xml:"evergreen"` - Description string `xml:"description"` - OutCue string `xml:"outcue"` - ISRC string `xml:"isrc"` - ISCI string `xml:"isci"` - Length int `xml:"length"` - OriginDateTime string `xml:"originDatetime"` - StartDateTime string `xml:"startDatetime"` - EndDateTime string `xml:"endDatetime"` - Sunday bool `xml:"sun"` - Monday bool `xml:"mon"` - Tuesday bool `xml:"tue"` - Wednesday bool `xml:"wed"` - Thursday bool `xml:"thu"` - Friday bool `xml:"fri"` - Saturday bool `xml:"sat"` - StartDaypart string `xml:"startDaypart"` - EndDayPart string `xml:"endDaypart"` - OriginName string `xml:"originName"` - Weight int `xml:"weight"` - LastPlayDateTime string `xml:"lastPlayDatetime"` - PlayCounter uint `xml:"playCounter"` - LocalCounter uint `xml:"localCounter"` - Validiy uint `xml:"validity"` - CondingFormat int `xml:"codingFormat"` - SampleRate int `xml:"sampleRate"` - BitRate int `xml:"bitRate"` - Channels int `xml:"channels"` - PlayGain int `xml:"playGain"` - StartPoint int `xml:"startPoint"` - EndPoint int `xml:"endPoint"` - FadeUpPoint int `xml:"fadeupPoint"` - FadeDownPoint int `xml:"fadedownPoint"` - SegueStartPoint int `xml:"segueStartPoint"` - SegueEndPoint int `xml:"segueEndPoint"` - SegueGain int `xml:"segueGain"` - HookStartPoint int `xml:"hookStartPoint"` - HookEndPoint int `xml:"hookEndPoint"` - TalkStartPoint int `xml:"talkStartPoint"` - TalkEndPoint int `xml:"talkEndPoint"` -} - -func NewRDCutAddFromXML(data io.Reader) (cut *RDCutAdd, err error) { - decoder := xml.NewDecoder(data) - cut = &RDCutAdd{} - if xmlerr := decoder.Decode(cut); xmlerr != nil { - err = fmt.Errorf("Error parsing XML response: %s", xmlerr) - return - } - return -} diff --git a/rhimport/conf.go b/rhimport/conf.go new file mode 100644 index 0000000..b7e8f88 --- /dev/null +++ b/rhimport/conf.go @@ -0,0 +1,51 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015-2016 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +type ImportParamDefaults struct { + Channels uint + NormalizationLevel int + AutotrimLevel int + UseMetaData bool +} + +type Config struct { + RDXportEndpoint string + TempDir string + LocalFetchDir string + ImportParamDefaults +} + +func NewConfig(rdxportEndpoint, tempDir, localFetchDir string) (conf *Config) { + conf = new(Config) + conf.RDXportEndpoint = rdxportEndpoint + conf.TempDir = tempDir + conf.LocalFetchDir = localFetchDir + conf.ImportParamDefaults.Channels = 2 + conf.ImportParamDefaults.NormalizationLevel = -12 + conf.ImportParamDefaults.AutotrimLevel = 0 + conf.ImportParamDefaults.UseMetaData = true + return +} diff --git a/rhimport/core.go b/rhimport/core.go new file mode 100644 index 0000000..98acd35 --- /dev/null +++ b/rhimport/core.go @@ -0,0 +1,192 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015-2016 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "fmt" + "github.com/andelf/go-curl" + "helsinki.at/rhrd-go/rddb" + "io/ioutil" + "log" + "os" +) + +const ( + CART_MAX = 999999 + CUT_MAX = 999 +) + +var ( + bool2str = map[bool]string{false: "0", true: "1"} + rhl = log.New(os.Stderr, "[rhimport]\t", log.LstdFlags) + rhdl = log.New(ioutil.Discard, "[rhimport-dbg]\t", log.LstdFlags) +) + +func init() { + if _, exists := os.LookupEnv("RHIMPORT_DEBUG"); exists { + rhdl.SetOutput(os.Stderr) + } + curl.GlobalInit(curl.GLOBAL_ALL) + fetcherInit() +} + +type ProgressCB func(step int, stepName string, progress float64, userdata interface{}) bool +type DoneCB func(Result, interface{}) bool + +type Result struct { + ResponseCode int + ErrorString string + Cart uint + Cut uint +} + +type Context struct { + conf *Config + db *rddb.DBChan + UserName string + Password string + Trusted bool + ShowId uint + ClearShowCarts bool + GroupName string + Cart uint + ClearCart bool + Cut uint + Channels uint + NormalizationLevel int + AutotrimLevel int + UseMetaData bool + SourceUri string + SourceFile string + DeleteSourceFile bool + DeleteSourceDir bool + ProgressCallBack ProgressCB + ProgressCallBackData interface{} + Cancel <-chan bool +} + +func NewContext(conf *Config, db *rddb.DBChan) *Context { + ctx := new(Context) + ctx.conf = conf + ctx.db = db + ctx.UserName = "" + ctx.Password = "" + ctx.Trusted = false + ctx.ShowId = 0 + ctx.ClearShowCarts = false + ctx.GroupName = "" + ctx.Cart = 0 + ctx.ClearCart = false + ctx.Cut = 0 + ctx.Channels = conf.ImportParamDefaults.Channels + ctx.NormalizationLevel = conf.ImportParamDefaults.NormalizationLevel + ctx.AutotrimLevel = conf.ImportParamDefaults.AutotrimLevel + ctx.UseMetaData = conf.ImportParamDefaults.UseMetaData + ctx.SourceFile = "" + ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false + ctx.ProgressCallBack = nil + ctx.Cancel = nil + + return ctx +} + +func (ctx *Context) SanityCheck() error { + if ctx.UserName == "" { + return fmt.Errorf("empty Username is not allowed") + } + if ctx.Password == "" && !ctx.Trusted { + return fmt.Errorf("empty Password on untrusted control interface is not allowed") + } + if ctx.ShowId != 0 { + if ctx.ShowId != 0 && ctx.ShowId > CART_MAX { + return fmt.Errorf("ShowId %d is outside of allowed range (0 < show-id < %d)", ctx.ShowId, CART_MAX) + } + if ctx.Cart != 0 && ctx.Cart > CART_MAX { + return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) + } + return nil + } + if ctx.GroupName != "" { + ismusic, err := ctx.checkMusicGroup() + if err != nil { + return err + } + if !ismusic { + return fmt.Errorf("supplied GroupName '%s' is not a music pool", ctx.GroupName) + } + if ctx.Cart != 0 || ctx.Cut != 0 { + return fmt.Errorf("Cart and Cut must not be supplied when importing into a music group") + } + return nil + } + if ctx.Cart == 0 { + return fmt.Errorf("either ShowId, PoolName or CartNumber must be supplied") + } + if ctx.Cart > CART_MAX { + return fmt.Errorf("Cart %d is outside of allowed range (0 < cart < %d)", ctx.Cart, CART_MAX) + } + if ctx.Cut != 0 && ctx.Cut > CUT_MAX { + return fmt.Errorf("Cut %d is outside of allowed range (0 < cart < %d)", ctx.Cut, CUT_MAX) + } + if ctx.Channels != 1 && ctx.Channels != 2 { + return fmt.Errorf("channles must be 1 or 2") + } + return nil +} + +func (ctx *Context) getPassword(cached bool) (err error) { + ctx.Password, err = ctx.db.GetPassword(ctx.UserName, cached) + return +} + +func (ctx *Context) CheckPassword() (bool, error) { + return ctx.db.CheckPassword(ctx.UserName, ctx.Password) +} + +func (ctx *Context) getGroupOfCart() (err error) { + ctx.GroupName, err = ctx.db.GetGroupOfCart(ctx.Cart) + return +} + +func (ctx *Context) getShowInfo() (carts []uint, err error) { + ctx.GroupName, ctx.NormalizationLevel, ctx.AutotrimLevel, carts, err = ctx.db.GetShowInfo(ctx.ShowId) + ctx.Channels = 2 + ctx.UseMetaData = true + return +} + +func (ctx *Context) checkMusicGroup() (bool, error) { + return ctx.db.CheckMusicGroup(ctx.GroupName) +} + +func (ctx *Context) getMusicInfo() (err error) { + ctx.NormalizationLevel, ctx.AutotrimLevel, err = ctx.db.GetMusicInfo(ctx.GroupName) + ctx.Channels = 2 + ctx.UseMetaData = true + ctx.Cart = 0 + ctx.Cut = 0 + return +} diff --git a/rhimport/fetcher.go b/rhimport/fetcher.go new file mode 100644 index 0000000..2d99be4 --- /dev/null +++ b/rhimport/fetcher.go @@ -0,0 +1,294 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015-2016 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "fmt" + "github.com/andelf/go-curl" + "io/ioutil" + "mime" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" +) + +type FetcherCurlCBData struct { + basepath string + filename string + remotename string + *os.File +} + +func (self *FetcherCurlCBData) Cleanup() { + if self.File != nil { + self.File.Close() + } +} + +func curlHeaderCallback(ptr []byte, userdata interface{}) bool { + hdr := fmt.Sprintf("%s", ptr) + data := userdata.(*FetcherCurlCBData) + + if strings.HasPrefix(hdr, "Content-Disposition:") { + if mediatype, params, err := mime.ParseMediaType(strings.TrimPrefix(hdr, "Content-Disposition:")); err == nil { + if mediatype == "attachment" { + data.filename = data.basepath + "/" + params["filename"] + } + } + } + return true +} + +func curlWriteCallback(ptr []byte, userdata interface{}) bool { + data := userdata.(*FetcherCurlCBData) + if data.File == nil { + if data.filename == "" { + data.filename = data.basepath + "/" + data.remotename + } + fp, err := os.OpenFile(data.filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + rhl.Printf("Unable to create file %s: %s", data.filename, err) + return false + } + data.File = fp + } + if _, err := data.File.Write(ptr); err != nil { + rhl.Printf("Unable to write file %s: %s", data.filename, err) + return false + } + return true +} + +func fetchFileCurl(ctx *Context, res *Result, uri *url.URL) (err error) { + rhl.Printf("curl-based fetcher called for '%s'", ctx.SourceUri) + + easy := curl.EasyInit() + if easy != nil { + defer easy.Cleanup() + + easy.Setopt(curl.OPT_FOLLOWLOCATION, true) + easy.Setopt(curl.OPT_URL, ctx.SourceUri) + + cbdata := &FetcherCurlCBData{remotename: path.Base(uri.Path)} + defer cbdata.Cleanup() + if cbdata.basepath, err = ioutil.TempDir(ctx.conf.TempDir, "rhimportd-"); err != nil { + return + } + + easy.Setopt(curl.OPT_HEADERFUNCTION, curlHeaderCallback) + easy.Setopt(curl.OPT_HEADERDATA, cbdata) + + easy.Setopt(curl.OPT_WRITEFUNCTION, curlWriteCallback) + easy.Setopt(curl.OPT_WRITEDATA, cbdata) + + easy.Setopt(curl.OPT_NOPROGRESS, false) + easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { + if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + rhl.Printf("downloading '%s' got canceled", ctx.SourceUri) + res.ResponseCode = http.StatusNoContent + res.ErrorString = "canceled" + return false + } + + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(1, "downloading", dlnow/dltotal, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + return true + }) + easy.Setopt(curl.OPT_PROGRESSDATA, ctx) + + if err = easy.Perform(); err != nil { + if cbdata.File != nil { + rhdl.Printf("Removing stale file: %s", cbdata.filename) + os.Remove(cbdata.filename) + os.Remove(path.Dir(cbdata.filename)) + } + if res.ResponseCode == http.StatusNoContent { + err = nil + } else { + err = fmt.Errorf("curl-fetcher('%s'): %s", ctx.SourceUri, err) + } + return + } + + ctx.SourceFile = cbdata.filename + ctx.DeleteSourceFile = true + ctx.DeleteSourceDir = true + } else { + err = fmt.Errorf("Error initializing libcurl") + } + + return +} + +func fetchFileLocal(ctx *Context, res *Result, uri *url.URL) (err error) { + rhl.Printf("Local fetcher called for '%s'", ctx.SourceUri) + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(1, "fetching", 0.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + + ctx.SourceFile = filepath.Join(ctx.conf.LocalFetchDir, path.Clean("/"+uri.Path)) + var src *os.File + if src, err = os.Open(ctx.SourceFile); err != nil { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("local-file open(): %s", err) + return nil + } + if info, err := src.Stat(); err != nil { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("local-file stat(): %s", err) + return nil + } else { + if info.IsDir() { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("'%s' is a directory", ctx.SourceFile) + return nil + } + } + src.Close() + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(1, "fetching", 1.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false + return +} + +func fetchFileFake(ctx *Context, res *Result, uri *url.URL) error { + rhdl.Printf("Fake fetcher for '%s'", ctx.SourceUri) + + if duration, err := strconv.ParseUint(uri.Host, 10, 32); err != nil { + err = nil + res.ResponseCode = http.StatusBadRequest + res.ErrorString = "invalid duration (must be a positive integer)" + } else { + for i := uint(0); i < uint(duration); i++ { + if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + rhl.Printf("faking got canceled") + res.ResponseCode = http.StatusNoContent + res.ErrorString = "canceled" + return nil + } + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(1, "faking", float64(i)/float64(duration), ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + time.Sleep(100 * time.Millisecond) + } + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(1, "faking", 1.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + ctx.SourceFile = "/nonexistend/fake.mp3" + ctx.DeleteSourceFile = false + ctx.DeleteSourceDir = false + } + return nil +} + +type FetchFunc func(*Context, *Result, *url.URL) (err error) + +// TODO: implement fetchers for: +// archiv:// +// public:// +// home:// ????? +var ( + fetchers = map[string]FetchFunc{ + "local": fetchFileLocal, + "fake": fetchFileFake, + } + curlProtos = map[string]bool{ + "http": false, "https": false, + "ftp": false, "ftps": false, + } +) + +func fetcherInit() { + info := curl.VersionInfo(curl.VERSION_FIRST) + protos := info.Protocols + for _, proto := range protos { + if _, ok := curlProtos[proto]; ok { + rhdl.Printf("curl: enabling protocol %s", proto) + fetchers[proto] = fetchFileCurl + curlProtos[proto] = true + } else { + rhdl.Printf("curl: ignoring protocol %s", proto) + } + } + for proto, enabled := range curlProtos { + if !enabled { + rhl.Printf("curl: protocol %s is disabled because the installed library version doesn't support it!", proto) + } + } +} + +func checkPassword(ctx *Context, result *Result) (err error) { + ok := false + if ok, err = ctx.CheckPassword(); err != nil { + return + } + if !ok { + result.ResponseCode = http.StatusUnauthorized + result.ErrorString = "invalid username and/or password" + } + return +} + +func FetchFile(ctx *Context) (res *Result, err error) { + res = &Result{ResponseCode: http.StatusOK} + + var uri *url.URL + if uri, err = url.Parse(ctx.SourceUri); err != nil { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("parsing uri: %s", err) + return res, nil + } + + if !ctx.Trusted { + if err = checkPassword(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + } + + if fetcher, ok := fetchers[uri.Scheme]; ok { + err = fetcher(ctx, res, uri) + } else { + err = fmt.Errorf("No fetcher for uri scheme '%s' found.", uri.Scheme) + } + return +} diff --git a/rhimport/importer.go b/rhimport/importer.go new file mode 100644 index 0000000..0702db4 --- /dev/null +++ b/rhimport/importer.go @@ -0,0 +1,528 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015-2016 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "bufio" + "bytes" + "fmt" + "github.com/andelf/go-curl" + "mime/multipart" + "net/http" + "os" + "path" +) + +func (self *Result) fromRDWebResult(rdres *RDWebResult) { + self.ResponseCode = rdres.ResponseCode + self.ErrorString = rdres.ErrorString + if rdres.AudioConvertError != 0 { + self.ErrorString += fmt.Sprintf(", Audio Convert Error: %d", rdres.AudioConvertError) + } +} + +func addCart(ctx *Context, res *Result) (err error) { + rhdl.Printf("importer: addCart() called for cart: %d", ctx.Cart) + + if ctx.GroupName == "" { + if err = ctx.getGroupOfCart(); err != nil { + return + } + } + + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "12"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("GROUP_NAME", ctx.GroupName); err != nil { + return + } + if err = w.WriteField("TYPE", "audio"); err != nil { + return + } + if ctx.Cart != 0 { + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + } + w.Close() + + var resp *http.Response + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { + return + } + res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + return + } + var cartadd *RDCartAdd + if cartadd, err = NewRDCartAddFromXML(resp.Body); err != nil { + return + } + res.ResponseCode = resp.StatusCode + res.ErrorString = "OK" + res.Cart = cartadd.Carts[0].Number + ctx.Cart = res.Cart + return +} + +func addCut(ctx *Context, res *Result) (err error) { + rhdl.Printf("importer: addCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "10"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + w.Close() + + var resp *http.Response + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { + return + } + res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + res.Cut = ctx.Cut + return + } + var cutadd *RDCutAdd + if cutadd, err = NewRDCutAddFromXML(resp.Body); err != nil { + return + } + res.ResponseCode = resp.StatusCode + res.ErrorString = "OK" + res.Cart = ctx.Cart + res.Cut = cutadd.Cuts[0].Number + ctx.Cut = cutadd.Cuts[0].Number + return +} + +func removeCart(ctx *Context, res *Result) (err error) { + rhdl.Printf("importer: removeCart() called for cart: %d", ctx.Cart) + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "13"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + w.Close() + + var resp *http.Response + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer resp.Body.Close() + + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { + return + } + res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + return +} + +func removeCut(ctx *Context, res *Result) (err error) { + rhdl.Printf("importer: removeCut() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) + var b bytes.Buffer + w := multipart.NewWriter(&b) + + if err = w.WriteField("COMMAND", "11"); err != nil { + return + } + if err = w.WriteField("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = w.WriteField("PASSWORD", ctx.Password); err != nil { + return + } + if err = w.WriteField("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + if err = w.WriteField("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { + return + } + w.Close() + + var resp *http.Response + if resp, err = sendPostRequest(ctx.conf.RDXportEndpoint, &b, w.FormDataContentType()); err != nil { + return + } + defer resp.Body.Close() + + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(resp.Body); err != nil { + return + } + res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + res.Cut = ctx.Cut + return +} + +func sendPostRequest(url string, b *bytes.Buffer, contenttype string) (resp *http.Response, err error) { + var req *http.Request + if req, err = http.NewRequest("POST", url, b); err != nil { + return + } + if contenttype != "" { + req.Header.Set("Content-Type", contenttype) + } + + client := &http.Client{} + if resp, err = client.Do(req); err != nil { + return + } + return +} + +func importAudioCreateRequest(ctx *Context, easy *curl.CURL) (form *curl.Form, err error) { + form = curl.NewForm() + + if err = form.Add("COMMAND", "2"); err != nil { + return + } + if err = form.Add("LOGIN_NAME", ctx.UserName); err != nil { + return + } + if err = form.Add("PASSWORD", ctx.Password); err != nil { + return + } + if err = form.Add("CART_NUMBER", fmt.Sprintf("%d", ctx.Cart)); err != nil { + return + } + if err = form.Add("CUT_NUMBER", fmt.Sprintf("%d", ctx.Cut)); err != nil { + return + } + if err = form.Add("CHANNELS", fmt.Sprintf("%d", ctx.Channels)); err != nil { + return + } + if err = form.Add("NORMALIZATION_LEVEL", fmt.Sprintf("%d", ctx.NormalizationLevel)); err != nil { + return + } + if err = form.Add("AUTOTRIM_LEVEL", fmt.Sprintf("%d", ctx.AutotrimLevel)); err != nil { + return + } + if err = form.Add("USE_METADATA", bool2str[ctx.UseMetaData]); err != nil { + return + } + if err = form.AddFile("FILENAME", ctx.SourceFile); err != nil { + return + } + + return +} + +func importAudio(ctx *Context, res *Result) (err error) { + rhdl.Printf("importer: importAudio() called for cart/cut: %d/%d", ctx.Cart, ctx.Cut) + easy := curl.EasyInit() + + if easy != nil { + defer easy.Cleanup() + + easy.Setopt(curl.OPT_URL, ctx.conf.RDXportEndpoint) + easy.Setopt(curl.OPT_POST, true) + + var form *curl.Form + if form, err = importAudioCreateRequest(ctx, easy); err != nil { + return + } + easy.Setopt(curl.OPT_HTTPPOST, form) + easy.Setopt(curl.OPT_HTTPHEADER, []string{"Expect:"}) + + var resbody bytes.Buffer + easy.Setopt(curl.OPT_WRITEFUNCTION, func(ptr []byte, userdata interface{}) bool { + b := userdata.(*bytes.Buffer) + b.Write(ptr) + return true + }) + easy.Setopt(curl.OPT_WRITEDATA, &resbody) + + easy.Setopt(curl.OPT_NOPROGRESS, false) + easy.Setopt(curl.OPT_PROGRESSFUNCTION, func(dltotal, dlnow, ultotal, ulnow float64, userdata interface{}) bool { + if ctx.Cancel != nil && len(ctx.Cancel) > 0 { + res.ResponseCode = http.StatusNoContent + res.ErrorString = "canceled" + return false + } + + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(2, "importing", ulnow/ultotal, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + return true + }) + easy.Setopt(curl.OPT_PROGRESSDATA, ctx) + + if err = easy.Perform(); err != nil { + if res.ResponseCode == http.StatusNoContent { + rhl.Printf("import to cart/cat %d/%d got canceled", ctx.Cart, ctx.Cut) + res.Cart = ctx.Cart + res.Cut = ctx.Cut + err = nil + } else { + err = fmt.Errorf("importer: %s", err) + } + return + } + + var rdres *RDWebResult + if rdres, err = NewRDWebResultFromXML(bufio.NewReader(&resbody)); err != nil { + return + } + res.fromRDWebResult(rdres) + res.Cart = ctx.Cart + res.Cut = ctx.Cut + } else { + err = fmt.Errorf("Error initializing libcurl") + } + + return +} + +func addCartCut(ctx *Context, res *Result) (err error) { + if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return removeCart(ctx, &Result{ResponseCode: http.StatusOK}) + } + return +} + +func removeAddCartCut(ctx *Context, res *Result) (err error) { + if err = removeCart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + return + } + return addCartCut(ctx, res) +} + +func isCartMemberOfShow(ctx *Context, res *Result, carts []uint) (found bool) { + if ctx.Cart == 0 { + return true + } + for _, cart := range carts { + if cart == ctx.Cart { + return true + } + } + res.ResponseCode = http.StatusBadRequest + res.ErrorString = fmt.Sprintf("Requested cart %d is not a member of show: %d", ctx.Cart, ctx.ShowId) + res.Cart = ctx.Cart + return false +} + +func clearShowCarts(ctx *Context, res *Result, carts []uint) (err error) { + if ctx.ClearShowCarts { + origCart := ctx.Cart + for _, cart := range carts { + ctx.Cart = cart + if err = removeCart(ctx, res); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + return + } + } + ctx.Cart = origCart + } + return +} + +func addShowCartCut(ctx *Context, res *Result, carts []uint) (err error) { + if err = addCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + for _, cart := range carts { + if cart == ctx.Cart { + if err = addCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return removeCart(ctx, &Result{ResponseCode: http.StatusOK}) + } + return + } + } + if err = removeCart(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + res.ResponseCode = http.StatusForbidden + res.ErrorString = fmt.Sprintf("Show %d has no free carts left", ctx.ShowId) + return +} + +func cleanupFiles(ctx *Context) { + if ctx.DeleteSourceFile { + rhdl.Printf("importer: removing file: %s", ctx.SourceFile) + if err := os.Remove(ctx.SourceFile); err != nil { + rhl.Printf("importer: error removing source file: %s", err) + return + } + if ctx.DeleteSourceDir { + dir := path.Dir(ctx.SourceFile) + rhdl.Printf("importer: also removing directory: %s", dir) + if err := os.Remove(dir); err != nil { + rhl.Printf("importer: error removing source directory: %s", err) + } + } + } + return +} + +func ImportFile(ctx *Context) (res *Result, err error) { + defer cleanupFiles(ctx) + + rhl.Printf("importer: ImportFile called with: show-id: %d, pool-name: '%s', cart/cut: %d/%d", ctx.ShowId, ctx.GroupName, ctx.Cart, ctx.Cut) + + if ctx.ProgressCallBack != nil { + if keep := ctx.ProgressCallBack(2, "importing", 0.0, ctx.ProgressCallBackData); !keep { + ctx.ProgressCallBack = nil + } + } + + // TODO: on trusted interfaces we should call getPassword again with cached=false after 401's + if ctx.Trusted { + if err = ctx.getPassword(true); err != nil { + return + } + } + + rmCartOnErr := false + rmCutOnErr := false + res = &Result{ResponseCode: http.StatusOK} + if ctx.ShowId != 0 { // Import to a show + var showCarts []uint + if showCarts, err = ctx.getShowInfo(); err != nil { + return + } + if !isCartMemberOfShow(ctx, res, showCarts) { + return + } + if err = clearShowCarts(ctx, res, showCarts); err != nil || (res.ResponseCode != http.StatusOK && res.ResponseCode != http.StatusNotFound) { + return + } + if ctx.ClearCart && !ctx.ClearShowCarts { + if err = removeAddCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + } else { + if err = addShowCartCut(ctx, res, showCarts); err != nil || res.ResponseCode != http.StatusOK { + return + } + } + rmCartOnErr = true + } else if ctx.GroupName != "" { // Import to music pool + if err = ctx.getMusicInfo(); err != nil { + return + } + if err = addCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + rmCartOnErr = true + } else if ctx.Cart != 0 && ctx.Cut == 0 { // Import to Cart + if ctx.ClearCart { + if err = removeAddCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + rmCartOnErr = true + } else { + if err = addCut(ctx, res); err != nil { + return + } + if res.ResponseCode != http.StatusOK { + if err = addCartCut(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + return + } + rmCartOnErr = true + } else { + rmCutOnErr = true + } + } + } + + if ctx.Cart != 0 && ctx.Cut != 0 { // Import to specific Cut within Cart + if err = importAudio(ctx, res); err != nil || res.ResponseCode != http.StatusOK { + if err != nil { + rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", ctx.Cart, ctx.Cut, err) + } else { + rhl.Printf("Fileimport has failed (Cart/Cut %d/%d): %s", res.Cart, res.Cut, res.ErrorString) + } + // Try to clean up after failed import + rmres := Result{ResponseCode: http.StatusOK} + if rmCartOnErr { + if rerr := removeCart(ctx, &rmres); rerr != nil { + return + } + } else if rmCutOnErr { + if rerr := removeCut(ctx, &rmres); rerr != nil { + return + } + } + } else { + rhl.Printf("File got succesfully imported into Cart/Cut %d/%d", res.Cart, res.Cut) + } + } else { + res.ResponseCode = http.StatusBadRequest + res.ErrorString = "importer: The request doesn't contain enough information to be processed" + } + + return +} diff --git a/rhimport/rdxport_responses.go b/rhimport/rdxport_responses.go new file mode 100644 index 0000000..2871408 --- /dev/null +++ b/rhimport/rdxport_responses.go @@ -0,0 +1,150 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015-2016 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "encoding/xml" + "fmt" + "io" +) + +type RDWebResult struct { + ResponseCode int `xml:"ResponseCode"` + ErrorString string `xml:"ErrorString"` + AudioConvertError int `xml:"AudioConvertError"` +} + +func NewRDWebResultFromXML(data io.Reader) (res *RDWebResult, err error) { + decoder := xml.NewDecoder(data) + res = &RDWebResult{} + if xmlerr := decoder.Decode(res); xmlerr != nil { + err = fmt.Errorf("Error parsing XML response: %s", xmlerr) + return + } + return +} + +type RDCartAdd struct { + Carts []RDCart `xml:"cart"` +} + +type RDCart struct { + Number uint `xml:"number"` + Type string `xml:"type"` + GroupName string `xml:"groupName"` + Title string `xml:"title"` + Artist string `xml:"artist"` + Album string `xml:"album"` + Year string `xml:"year"` + Label string `xml:"label"` + Client string `xml:"client"` + Agency string `xml:"agency"` + Publisher string `xml:"publisher"` + Composer string `xml:"composer"` + UserDefined string `xml:"userDefined"` + UsageCode uint `xml:"usageCode"` + ForcedLength string `xml:"forcedLength"` + AverageLength string `xml:"averageLength"` + LengthDeviation string `xml:"lengthDeviation"` + AverageSegueLength string `xml:"averageSegueLenth"` + AverageHookLength string `xml:"averageHookLength"` + CutQuantity uint `xml:"cutQuantity"` + LastCutPlayed uint `xml:"lastCutPlayed"` + Validity uint `xml:"validity"` + EnforceLength bool `xml:"enforceLength"` + Asynchronous bool `xml:"asyncronous"` + Owner string `xml:"owner"` + MetadataDatetime string `xml:"metadataDatetime"` +} + +func NewRDCartAddFromXML(data io.Reader) (cart *RDCartAdd, err error) { + decoder := xml.NewDecoder(data) + cart = &RDCartAdd{} + if xmlerr := decoder.Decode(cart); xmlerr != nil { + err = fmt.Errorf("Error parsing XML response: %s", xmlerr) + return + } + return +} + +type RDCutAdd struct { + Cuts []RDCut `xml:"cut"` +} + +type RDCut struct { + Name string `xml:"cutName"` + CartNumber uint `xml:"cartNumber"` + Number uint `xml:"cutNumber"` + EverGreen bool `xml:"evergreen"` + Description string `xml:"description"` + OutCue string `xml:"outcue"` + ISRC string `xml:"isrc"` + ISCI string `xml:"isci"` + Length int `xml:"length"` + OriginDateTime string `xml:"originDatetime"` + StartDateTime string `xml:"startDatetime"` + EndDateTime string `xml:"endDatetime"` + Sunday bool `xml:"sun"` + Monday bool `xml:"mon"` + Tuesday bool `xml:"tue"` + Wednesday bool `xml:"wed"` + Thursday bool `xml:"thu"` + Friday bool `xml:"fri"` + Saturday bool `xml:"sat"` + StartDaypart string `xml:"startDaypart"` + EndDayPart string `xml:"endDaypart"` + OriginName string `xml:"originName"` + Weight int `xml:"weight"` + LastPlayDateTime string `xml:"lastPlayDatetime"` + PlayCounter uint `xml:"playCounter"` + LocalCounter uint `xml:"localCounter"` + Validiy uint `xml:"validity"` + CondingFormat int `xml:"codingFormat"` + SampleRate int `xml:"sampleRate"` + BitRate int `xml:"bitRate"` + Channels int `xml:"channels"` + PlayGain int `xml:"playGain"` + StartPoint int `xml:"startPoint"` + EndPoint int `xml:"endPoint"` + FadeUpPoint int `xml:"fadeupPoint"` + FadeDownPoint int `xml:"fadedownPoint"` + SegueStartPoint int `xml:"segueStartPoint"` + SegueEndPoint int `xml:"segueEndPoint"` + SegueGain int `xml:"segueGain"` + HookStartPoint int `xml:"hookStartPoint"` + HookEndPoint int `xml:"hookEndPoint"` + TalkStartPoint int `xml:"talkStartPoint"` + TalkEndPoint int `xml:"talkEndPoint"` +} + +func NewRDCutAddFromXML(data io.Reader) (cut *RDCutAdd, err error) { + decoder := xml.NewDecoder(data) + cut = &RDCutAdd{} + if xmlerr := decoder.Decode(cut); xmlerr != nil { + err = fmt.Errorf("Error parsing XML response: %s", xmlerr) + return + } + return +} diff --git a/rhimport/session.go b/rhimport/session.go new file mode 100644 index 0000000..66705ec --- /dev/null +++ b/rhimport/session.go @@ -0,0 +1,328 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015-2016 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "fmt" + "net/http" + "time" +) + +const ( + SESSION_NEW = iota + SESSION_RUNNING + SESSION_CANCELED + SESSION_DONE + SESSION_TIMEOUT +) + +type Session struct { + ctx Context + state int + removeFunc func() + done chan bool + quit chan bool + timer *time.Timer + cancelIntChan chan bool + progressIntChan chan ProgressData + doneIntChan chan Result + runChan chan time.Duration + cancelChan chan bool + addProgressChan chan sessionAddProgressHandlerRequest + addDoneChan chan sessionAddDoneHandlerRequest + progressCBs []*SessionProgressCB + doneCBs []*SessionDoneCB +} + +type SessionProgressCB struct { + cb ProgressCB + userdata interface{} +} + +type SessionDoneCB struct { + cb DoneCB + userdata interface{} +} + +type ProgressData struct { + Step int + StepName string + Progress float64 +} + +type sessionAddProgressHandlerResponse struct { + err error +} + +type sessionAddProgressHandlerRequest struct { + userdata interface{} + callback ProgressCB + response chan<- sessionAddProgressHandlerResponse +} + +type sessionAddDoneHandlerResponse struct { + err error +} + +type sessionAddDoneHandlerRequest struct { + userdata interface{} + callback DoneCB + response chan<- sessionAddDoneHandlerResponse +} + +func sessionProgressCallback(step int, stepName string, progress float64, userdata interface{}) bool { + out := userdata.(chan<- ProgressData) + out <- ProgressData{step, stepName, progress} + return true +} + +func sessionRun(ctx Context, done chan<- Result) { + if err := ctx.SanityCheck(); err != nil { + done <- Result{http.StatusBadRequest, err.Error(), 0, 0} + return + } + + if res, err := FetchFile(&ctx); err != nil { + done <- Result{http.StatusInternalServerError, err.Error(), 0, 0} + return + } else if res.ResponseCode != http.StatusOK { + done <- *res + return + } + + if res, err := ImportFile(&ctx); err != nil { + done <- Result{http.StatusInternalServerError, err.Error(), 0, 0} + return + } else { + done <- *res + return + } +} + +func (self *Session) run(timeout time.Duration) { + self.ctx.ProgressCallBack = sessionProgressCallback + self.ctx.ProgressCallBackData = (chan<- ProgressData)(self.progressIntChan) + self.ctx.Cancel = self.cancelIntChan + go sessionRun(self.ctx, self.doneIntChan) + self.state = SESSION_RUNNING + self.timer.Reset(timeout) + return +} + +func (self *Session) cancel() { + rhdl.Println("Session: canceling running import") + select { + case self.cancelIntChan <- true: + default: // session got canceled already?? + } + self.state = SESSION_CANCELED +} + +func (self *Session) addProgressHandler(userdata interface{}, cb ProgressCB) (resp sessionAddProgressHandlerResponse) { + if self.state != SESSION_NEW && self.state != SESSION_RUNNING { + resp.err = fmt.Errorf("session is already done/canceled") + } + self.progressCBs = append(self.progressCBs, &SessionProgressCB{cb, userdata}) + return +} + +func (self *Session) addDoneHandler(userdata interface{}, cb DoneCB) (resp sessionAddDoneHandlerResponse) { + if self.state != SESSION_NEW && self.state != SESSION_RUNNING { + resp.err = fmt.Errorf("session is already done/canceled") + } + self.doneCBs = append(self.doneCBs, &SessionDoneCB{cb, userdata}) + return +} + +func (self *Session) callProgressHandler(p *ProgressData) { + for _, cb := range self.progressCBs { + if cb.cb != nil { + if keep := cb.cb(p.Step, p.StepName, p.Progress, cb.userdata); !keep { + cb.cb = nil + } + } + } +} + +func (self *Session) callDoneHandler(r *Result) { + for _, cb := range self.doneCBs { + if cb.cb != nil { + if keep := cb.cb(*r, cb.userdata); !keep { + cb.cb = nil + } + } + } +} + +func (self *Session) dispatchRequests() { + defer func() { self.done <- true }() + for { + select { + case <-self.quit: + if self.state == SESSION_RUNNING { + self.cancel() + } + return + case <-self.timer.C: + if self.state == SESSION_RUNNING { + self.cancel() + } + self.state = SESSION_TIMEOUT + r := &Result{500, "session timed out", 0, 0} + self.callDoneHandler(r) + if self.removeFunc != nil { + self.removeFunc() + } + case t := <-self.runChan: + if self.state == SESSION_NEW { + self.run(t) + } + case <-self.cancelChan: + if self.state == SESSION_RUNNING { + self.cancel() + } + case req := <-self.addProgressChan: + req.response <- self.addProgressHandler(req.userdata, req.callback) + case req := <-self.addDoneChan: + req.response <- self.addDoneHandler(req.userdata, req.callback) + case p := <-self.progressIntChan: + self.callProgressHandler(&p) + case r := <-self.doneIntChan: + if self.state != SESSION_TIMEOUT { + self.timer.Stop() + self.state = SESSION_DONE + self.callDoneHandler(&r) + if self.removeFunc != nil { + self.removeFunc() + } + } + } + } +} + +// ********************************************************* +// Public Interface + +type SessionChan struct { + runChan chan<- time.Duration + cancelChan chan<- bool + addProgressChan chan<- sessionAddProgressHandlerRequest + addDoneChan chan<- sessionAddDoneHandlerRequest +} + +func (self *SessionChan) Run(timeout time.Duration) { + select { + case self.runChan <- timeout: + default: // command is already pending or session is about to be closed/removed + } +} + +func (self *SessionChan) Cancel() { + select { + case self.cancelChan <- true: + default: // cancel is already pending or session is about to be closed/removed + } +} + +func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ProgressCB) error { + resCh := make(chan sessionAddProgressHandlerResponse) + req := sessionAddProgressHandlerRequest{} + req.userdata = userdata + req.callback = cb + req.response = resCh + select { + case self.addProgressChan <- req: + default: + return fmt.Errorf("session is about to be closed/removed") + } + + res := <-resCh + return res.err +} + +func (self *SessionChan) AddDoneHandler(userdata interface{}, cb DoneCB) error { + resCh := make(chan sessionAddDoneHandlerResponse) + req := sessionAddDoneHandlerRequest{} + req.userdata = userdata + req.callback = cb + req.response = resCh + select { + case self.addDoneChan <- req: + default: + return fmt.Errorf("session is about to be closed/removed") + } + + res := <-resCh + return res.err +} + +// ********************************************************* +// Semi-Public Interface (only used by sessionStore) + +func (self *Session) getInterface() *SessionChan { + ch := &SessionChan{} + ch.runChan = self.runChan + ch.cancelChan = self.cancelChan + ch.addProgressChan = self.addProgressChan + ch.addDoneChan = self.addDoneChan + return ch +} + +func (self *Session) cleanup() { + self.quit <- true + rhdl.Printf("waiting for session to close") + <-self.done + close(self.quit) + close(self.done) + self.timer.Stop() + // don't close the channels we give out because this might lead to a panic if + // somebody wites to an already removed session + // close(self.cancelIntChan) + // close(self.progressIntChan) + // close(self.doneIntChan) + // close(self.runChan) + // close(self.cancelChan) + // close(self.addProgressChan) + // close(self.addDoneChan) + rhdl.Printf("session is now cleaned up") +} + +func newSession(ctx *Context, removeFunc func()) (session *Session) { + session = new(Session) + session.state = SESSION_NEW + session.removeFunc = removeFunc + session.ctx = *ctx + session.done = make(chan bool) + session.timer = time.NewTimer(10 * time.Second) + session.cancelIntChan = make(chan bool, 1) + session.progressIntChan = make(chan ProgressData, 10) + session.doneIntChan = make(chan Result, 1) + session.runChan = make(chan time.Duration, 1) + session.cancelChan = make(chan bool, 1) + session.addProgressChan = make(chan sessionAddProgressHandlerRequest, 10) + session.addDoneChan = make(chan sessionAddDoneHandlerRequest, 10) + go session.dispatchRequests() + return +} diff --git a/rhimport/session_store.go b/rhimport/session_store.go new file mode 100644 index 0000000..e47366e --- /dev/null +++ b/rhimport/session_store.go @@ -0,0 +1,310 @@ +// +// rhimportd +// +// The Radio Helsinki Rivendell Import Daemon +// +// +// Copyright (C) 2015-2016 Christian Pointner +// +// This file is part of rhimportd. +// +// rhimportd is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// any later version. +// +// rhimportd is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with rhimportd. If not, see . +// + +package rhimport + +import ( + "crypto/rand" + "encoding/base64" + "fmt" + "helsinki.at/rhrd-go/rddb" + "net/http" +) + +type newSessionResponse struct { + id string + session *SessionChan + responsecode int + errorstring string +} + +type newSessionRequest struct { + ctx *Context + refId string + response chan newSessionResponse +} + +type getSessionResponse struct { + session *SessionChan + refId string + responsecode int + errorstring string +} + +type getSessionRequest struct { + user string + id string + refId string + response chan getSessionResponse +} + +type listSessionsResponse struct { + sessions map[string]string + responsecode int + errorstring string +} + +type listSessionsRequest struct { + user string + password string + trusted bool + response chan listSessionsResponse +} + +type removeSessionResponse struct { + responsecode int + errorstring string +} + +type removeSessionRequest struct { + user string + id string + response chan removeSessionResponse +} + +type SessionStoreElement struct { + s *Session + refId string +} + +type SessionStore struct { + store map[string]map[string]*SessionStoreElement + conf *Config + db *rddb.DBChan + quit chan bool + done chan bool + newChan chan newSessionRequest + getChan chan getSessionRequest + listChan chan listSessionsRequest + removeChan chan removeSessionRequest +} + +func generateSessionId() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return base64.RawStdEncoding.EncodeToString(b[:]), nil +} + +func (self *SessionStore) new(ctx *Context, refId string) (resp newSessionResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" + if !ctx.Trusted { + if ok, err := self.db.CheckPassword(ctx.UserName, ctx.Password); err != nil { + resp.responsecode = http.StatusInternalServerError + resp.errorstring = err.Error() + return + } else if !ok { + resp.responsecode = http.StatusUnauthorized + resp.errorstring = "invalid username and/or password" + return + } + } + if id, err := generateSessionId(); err != nil { + resp.responsecode = http.StatusInternalServerError + resp.errorstring = err.Error() + } else { + resp.id = id + if _, exists := self.store[ctx.UserName]; !exists { + self.store[ctx.UserName] = make(map[string]*SessionStoreElement) + } + ctx.conf = self.conf + ctx.db = self.db + s := &SessionStoreElement{newSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }), refId} + self.store[ctx.UserName][resp.id] = s + resp.session = self.store[ctx.UserName][resp.id].s.getInterface() + rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) + } + return +} + +func (self *SessionStore) get(user, id string) (resp getSessionResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" + if session, exists := self.store[user][id]; exists { + resp.session = session.s.getInterface() + resp.refId = session.refId + } else { + resp.responsecode = http.StatusNotFound + resp.errorstring = fmt.Sprintf("SessionStore: session '%s/%s' not found", user, id) + } + return +} + +func (self *SessionStore) list(user, password string, trusted bool) (resp listSessionsResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" + if !trusted { + if ok, err := self.db.CheckPassword(user, password); err != nil { + resp.responsecode = http.StatusInternalServerError + resp.errorstring = err.Error() + return + } else if !ok { + resp.responsecode = http.StatusUnauthorized + resp.errorstring = "invalid username and/or password" + return + } + } + resp.sessions = make(map[string]string) + if sessions, exists := self.store[user]; exists { + for id, e := range sessions { + resp.sessions[id] = e.refId + } + } + return +} + +func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { + resp.responsecode = http.StatusOK + resp.errorstring = "OK" + if session, exists := self.store[user][id]; exists { + go session.s.cleanup() // cleanup could take a while -> don't block all the other stuff + delete(self.store[user], id) + rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) + if userstore, exists := self.store[user]; exists { + if len(userstore) == 0 { + delete(self.store, user) + rhdl.Printf("SessionStore: removed user '%s'", user) + } + } + } else { + resp.responsecode = http.StatusNotFound + resp.errorstring = fmt.Sprintf("SessionStore: session '%s/%s' not found", user, id) + } + return +} + +func (self *SessionStore) dispatchRequests() { + defer func() { self.done <- true }() + for { + select { + case <-self.quit: + return + case req := <-self.newChan: + req.response <- self.new(req.ctx, req.refId) + case req := <-self.getChan: + req.response <- self.get(req.user, req.id) + case req := <-self.listChan: + req.response <- self.list(req.user, req.password, req.trusted) + case req := <-self.removeChan: + req.response <- self.remove(req.user, req.id) + } + } +} + +// ********************************************************* +// Public Interface + +type SessionStoreChan struct { + newChan chan<- newSessionRequest + getChan chan<- getSessionRequest + listChan chan listSessionsRequest + removeChan chan<- removeSessionRequest +} + +func (self *SessionStoreChan) New(ctx *Context, refId string) (string, *SessionChan, int, string) { + resCh := make(chan newSessionResponse) + req := newSessionRequest{} + req.ctx = ctx + req.refId = refId + req.response = resCh + self.newChan <- req + + res := <-resCh + return res.id, res.session, res.responsecode, res.errorstring +} + +func (self *SessionStoreChan) Get(user, id string) (*SessionChan, string, int, string) { + resCh := make(chan getSessionResponse) + req := getSessionRequest{} + req.user = user + req.id = id + req.response = resCh + self.getChan <- req + + res := <-resCh + return res.session, res.refId, res.responsecode, res.errorstring +} + +func (self *SessionStoreChan) List(user, password string, trusted bool) (map[string]string, int, string) { + resCh := make(chan listSessionsResponse) + req := listSessionsRequest{} + req.user = user + req.password = password + req.trusted = trusted + req.response = resCh + self.listChan <- req + + res := <-resCh + return res.sessions, res.responsecode, res.errorstring +} + +func (self *SessionStoreChan) Remove(user, id string) (int, string) { + resCh := make(chan removeSessionResponse) + req := removeSessionRequest{} + req.user = user + req.id = id + req.response = resCh + self.removeChan <- req + + res := <-resCh + return res.responsecode, res.errorstring +} + +func (self *SessionStore) GetInterface() *SessionStoreChan { + ch := &SessionStoreChan{} + ch.newChan = self.newChan + ch.getChan = self.getChan + ch.listChan = self.listChan + ch.removeChan = self.removeChan + return ch +} + +func (self *SessionStore) Cleanup() { + self.quit <- true + <-self.done + close(self.quit) + close(self.done) + close(self.newChan) + close(self.getChan) + close(self.listChan) + close(self.removeChan) +} + +func NewSessionStore(conf *Config, db *rddb.DBChan) (store *SessionStore, err error) { + store = new(SessionStore) + store.conf = conf + store.db = db + store.quit = make(chan bool) + store.done = make(chan bool) + store.store = make(map[string]map[string]*SessionStoreElement) + store.newChan = make(chan newSessionRequest, 10) + store.getChan = make(chan getSessionRequest, 10) + store.listChan = make(chan listSessionsRequest, 10) + store.removeChan = make(chan removeSessionRequest, 10) + + go store.dispatchRequests() + return +} diff --git a/session.go b/session.go deleted file mode 100644 index 66705ec..0000000 --- a/session.go +++ /dev/null @@ -1,328 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "fmt" - "net/http" - "time" -) - -const ( - SESSION_NEW = iota - SESSION_RUNNING - SESSION_CANCELED - SESSION_DONE - SESSION_TIMEOUT -) - -type Session struct { - ctx Context - state int - removeFunc func() - done chan bool - quit chan bool - timer *time.Timer - cancelIntChan chan bool - progressIntChan chan ProgressData - doneIntChan chan Result - runChan chan time.Duration - cancelChan chan bool - addProgressChan chan sessionAddProgressHandlerRequest - addDoneChan chan sessionAddDoneHandlerRequest - progressCBs []*SessionProgressCB - doneCBs []*SessionDoneCB -} - -type SessionProgressCB struct { - cb ProgressCB - userdata interface{} -} - -type SessionDoneCB struct { - cb DoneCB - userdata interface{} -} - -type ProgressData struct { - Step int - StepName string - Progress float64 -} - -type sessionAddProgressHandlerResponse struct { - err error -} - -type sessionAddProgressHandlerRequest struct { - userdata interface{} - callback ProgressCB - response chan<- sessionAddProgressHandlerResponse -} - -type sessionAddDoneHandlerResponse struct { - err error -} - -type sessionAddDoneHandlerRequest struct { - userdata interface{} - callback DoneCB - response chan<- sessionAddDoneHandlerResponse -} - -func sessionProgressCallback(step int, stepName string, progress float64, userdata interface{}) bool { - out := userdata.(chan<- ProgressData) - out <- ProgressData{step, stepName, progress} - return true -} - -func sessionRun(ctx Context, done chan<- Result) { - if err := ctx.SanityCheck(); err != nil { - done <- Result{http.StatusBadRequest, err.Error(), 0, 0} - return - } - - if res, err := FetchFile(&ctx); err != nil { - done <- Result{http.StatusInternalServerError, err.Error(), 0, 0} - return - } else if res.ResponseCode != http.StatusOK { - done <- *res - return - } - - if res, err := ImportFile(&ctx); err != nil { - done <- Result{http.StatusInternalServerError, err.Error(), 0, 0} - return - } else { - done <- *res - return - } -} - -func (self *Session) run(timeout time.Duration) { - self.ctx.ProgressCallBack = sessionProgressCallback - self.ctx.ProgressCallBackData = (chan<- ProgressData)(self.progressIntChan) - self.ctx.Cancel = self.cancelIntChan - go sessionRun(self.ctx, self.doneIntChan) - self.state = SESSION_RUNNING - self.timer.Reset(timeout) - return -} - -func (self *Session) cancel() { - rhdl.Println("Session: canceling running import") - select { - case self.cancelIntChan <- true: - default: // session got canceled already?? - } - self.state = SESSION_CANCELED -} - -func (self *Session) addProgressHandler(userdata interface{}, cb ProgressCB) (resp sessionAddProgressHandlerResponse) { - if self.state != SESSION_NEW && self.state != SESSION_RUNNING { - resp.err = fmt.Errorf("session is already done/canceled") - } - self.progressCBs = append(self.progressCBs, &SessionProgressCB{cb, userdata}) - return -} - -func (self *Session) addDoneHandler(userdata interface{}, cb DoneCB) (resp sessionAddDoneHandlerResponse) { - if self.state != SESSION_NEW && self.state != SESSION_RUNNING { - resp.err = fmt.Errorf("session is already done/canceled") - } - self.doneCBs = append(self.doneCBs, &SessionDoneCB{cb, userdata}) - return -} - -func (self *Session) callProgressHandler(p *ProgressData) { - for _, cb := range self.progressCBs { - if cb.cb != nil { - if keep := cb.cb(p.Step, p.StepName, p.Progress, cb.userdata); !keep { - cb.cb = nil - } - } - } -} - -func (self *Session) callDoneHandler(r *Result) { - for _, cb := range self.doneCBs { - if cb.cb != nil { - if keep := cb.cb(*r, cb.userdata); !keep { - cb.cb = nil - } - } - } -} - -func (self *Session) dispatchRequests() { - defer func() { self.done <- true }() - for { - select { - case <-self.quit: - if self.state == SESSION_RUNNING { - self.cancel() - } - return - case <-self.timer.C: - if self.state == SESSION_RUNNING { - self.cancel() - } - self.state = SESSION_TIMEOUT - r := &Result{500, "session timed out", 0, 0} - self.callDoneHandler(r) - if self.removeFunc != nil { - self.removeFunc() - } - case t := <-self.runChan: - if self.state == SESSION_NEW { - self.run(t) - } - case <-self.cancelChan: - if self.state == SESSION_RUNNING { - self.cancel() - } - case req := <-self.addProgressChan: - req.response <- self.addProgressHandler(req.userdata, req.callback) - case req := <-self.addDoneChan: - req.response <- self.addDoneHandler(req.userdata, req.callback) - case p := <-self.progressIntChan: - self.callProgressHandler(&p) - case r := <-self.doneIntChan: - if self.state != SESSION_TIMEOUT { - self.timer.Stop() - self.state = SESSION_DONE - self.callDoneHandler(&r) - if self.removeFunc != nil { - self.removeFunc() - } - } - } - } -} - -// ********************************************************* -// Public Interface - -type SessionChan struct { - runChan chan<- time.Duration - cancelChan chan<- bool - addProgressChan chan<- sessionAddProgressHandlerRequest - addDoneChan chan<- sessionAddDoneHandlerRequest -} - -func (self *SessionChan) Run(timeout time.Duration) { - select { - case self.runChan <- timeout: - default: // command is already pending or session is about to be closed/removed - } -} - -func (self *SessionChan) Cancel() { - select { - case self.cancelChan <- true: - default: // cancel is already pending or session is about to be closed/removed - } -} - -func (self *SessionChan) AddProgressHandler(userdata interface{}, cb ProgressCB) error { - resCh := make(chan sessionAddProgressHandlerResponse) - req := sessionAddProgressHandlerRequest{} - req.userdata = userdata - req.callback = cb - req.response = resCh - select { - case self.addProgressChan <- req: - default: - return fmt.Errorf("session is about to be closed/removed") - } - - res := <-resCh - return res.err -} - -func (self *SessionChan) AddDoneHandler(userdata interface{}, cb DoneCB) error { - resCh := make(chan sessionAddDoneHandlerResponse) - req := sessionAddDoneHandlerRequest{} - req.userdata = userdata - req.callback = cb - req.response = resCh - select { - case self.addDoneChan <- req: - default: - return fmt.Errorf("session is about to be closed/removed") - } - - res := <-resCh - return res.err -} - -// ********************************************************* -// Semi-Public Interface (only used by sessionStore) - -func (self *Session) getInterface() *SessionChan { - ch := &SessionChan{} - ch.runChan = self.runChan - ch.cancelChan = self.cancelChan - ch.addProgressChan = self.addProgressChan - ch.addDoneChan = self.addDoneChan - return ch -} - -func (self *Session) cleanup() { - self.quit <- true - rhdl.Printf("waiting for session to close") - <-self.done - close(self.quit) - close(self.done) - self.timer.Stop() - // don't close the channels we give out because this might lead to a panic if - // somebody wites to an already removed session - // close(self.cancelIntChan) - // close(self.progressIntChan) - // close(self.doneIntChan) - // close(self.runChan) - // close(self.cancelChan) - // close(self.addProgressChan) - // close(self.addDoneChan) - rhdl.Printf("session is now cleaned up") -} - -func newSession(ctx *Context, removeFunc func()) (session *Session) { - session = new(Session) - session.state = SESSION_NEW - session.removeFunc = removeFunc - session.ctx = *ctx - session.done = make(chan bool) - session.timer = time.NewTimer(10 * time.Second) - session.cancelIntChan = make(chan bool, 1) - session.progressIntChan = make(chan ProgressData, 10) - session.doneIntChan = make(chan Result, 1) - session.runChan = make(chan time.Duration, 1) - session.cancelChan = make(chan bool, 1) - session.addProgressChan = make(chan sessionAddProgressHandlerRequest, 10) - session.addDoneChan = make(chan sessionAddDoneHandlerRequest, 10) - go session.dispatchRequests() - return -} diff --git a/session_store.go b/session_store.go deleted file mode 100644 index e47366e..0000000 --- a/session_store.go +++ /dev/null @@ -1,310 +0,0 @@ -// -// rhimportd -// -// The Radio Helsinki Rivendell Import Daemon -// -// -// Copyright (C) 2015-2016 Christian Pointner -// -// This file is part of rhimportd. -// -// rhimportd is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// rhimportd is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with rhimportd. If not, see . -// - -package rhimport - -import ( - "crypto/rand" - "encoding/base64" - "fmt" - "helsinki.at/rhrd-go/rddb" - "net/http" -) - -type newSessionResponse struct { - id string - session *SessionChan - responsecode int - errorstring string -} - -type newSessionRequest struct { - ctx *Context - refId string - response chan newSessionResponse -} - -type getSessionResponse struct { - session *SessionChan - refId string - responsecode int - errorstring string -} - -type getSessionRequest struct { - user string - id string - refId string - response chan getSessionResponse -} - -type listSessionsResponse struct { - sessions map[string]string - responsecode int - errorstring string -} - -type listSessionsRequest struct { - user string - password string - trusted bool - response chan listSessionsResponse -} - -type removeSessionResponse struct { - responsecode int - errorstring string -} - -type removeSessionRequest struct { - user string - id string - response chan removeSessionResponse -} - -type SessionStoreElement struct { - s *Session - refId string -} - -type SessionStore struct { - store map[string]map[string]*SessionStoreElement - conf *Config - db *rddb.DBChan - quit chan bool - done chan bool - newChan chan newSessionRequest - getChan chan getSessionRequest - listChan chan listSessionsRequest - removeChan chan removeSessionRequest -} - -func generateSessionId() (string, error) { - var b [32]byte - if _, err := rand.Read(b[:]); err != nil { - return "", err - } - return base64.RawStdEncoding.EncodeToString(b[:]), nil -} - -func (self *SessionStore) new(ctx *Context, refId string) (resp newSessionResponse) { - resp.responsecode = http.StatusOK - resp.errorstring = "OK" - if !ctx.Trusted { - if ok, err := self.db.CheckPassword(ctx.UserName, ctx.Password); err != nil { - resp.responsecode = http.StatusInternalServerError - resp.errorstring = err.Error() - return - } else if !ok { - resp.responsecode = http.StatusUnauthorized - resp.errorstring = "invalid username and/or password" - return - } - } - if id, err := generateSessionId(); err != nil { - resp.responsecode = http.StatusInternalServerError - resp.errorstring = err.Error() - } else { - resp.id = id - if _, exists := self.store[ctx.UserName]; !exists { - self.store[ctx.UserName] = make(map[string]*SessionStoreElement) - } - ctx.conf = self.conf - ctx.db = self.db - s := &SessionStoreElement{newSession(ctx, func() { self.GetInterface().Remove(ctx.UserName, resp.id) }), refId} - self.store[ctx.UserName][resp.id] = s - resp.session = self.store[ctx.UserName][resp.id].s.getInterface() - rhdl.Printf("SessionStore: created session for '%s' -> %s", ctx.UserName, resp.id) - } - return -} - -func (self *SessionStore) get(user, id string) (resp getSessionResponse) { - resp.responsecode = http.StatusOK - resp.errorstring = "OK" - if session, exists := self.store[user][id]; exists { - resp.session = session.s.getInterface() - resp.refId = session.refId - } else { - resp.responsecode = http.StatusNotFound - resp.errorstring = fmt.Sprintf("SessionStore: session '%s/%s' not found", user, id) - } - return -} - -func (self *SessionStore) list(user, password string, trusted bool) (resp listSessionsResponse) { - resp.responsecode = http.StatusOK - resp.errorstring = "OK" - if !trusted { - if ok, err := self.db.CheckPassword(user, password); err != nil { - resp.responsecode = http.StatusInternalServerError - resp.errorstring = err.Error() - return - } else if !ok { - resp.responsecode = http.StatusUnauthorized - resp.errorstring = "invalid username and/or password" - return - } - } - resp.sessions = make(map[string]string) - if sessions, exists := self.store[user]; exists { - for id, e := range sessions { - resp.sessions[id] = e.refId - } - } - return -} - -func (self *SessionStore) remove(user, id string) (resp removeSessionResponse) { - resp.responsecode = http.StatusOK - resp.errorstring = "OK" - if session, exists := self.store[user][id]; exists { - go session.s.cleanup() // cleanup could take a while -> don't block all the other stuff - delete(self.store[user], id) - rhdl.Printf("SessionStore: removed session '%s/%s'", user, id) - if userstore, exists := self.store[user]; exists { - if len(userstore) == 0 { - delete(self.store, user) - rhdl.Printf("SessionStore: removed user '%s'", user) - } - } - } else { - resp.responsecode = http.StatusNotFound - resp.errorstring = fmt.Sprintf("SessionStore: session '%s/%s' not found", user, id) - } - return -} - -func (self *SessionStore) dispatchRequests() { - defer func() { self.done <- true }() - for { - select { - case <-self.quit: - return - case req := <-self.newChan: - req.response <- self.new(req.ctx, req.refId) - case req := <-self.getChan: - req.response <- self.get(req.user, req.id) - case req := <-self.listChan: - req.response <- self.list(req.user, req.password, req.trusted) - case req := <-self.removeChan: - req.response <- self.remove(req.user, req.id) - } - } -} - -// ********************************************************* -// Public Interface - -type SessionStoreChan struct { - newChan chan<- newSessionRequest - getChan chan<- getSessionRequest - listChan chan listSessionsRequest - removeChan chan<- removeSessionRequest -} - -func (self *SessionStoreChan) New(ctx *Context, refId string) (string, *SessionChan, int, string) { - resCh := make(chan newSessionResponse) - req := newSessionRequest{} - req.ctx = ctx - req.refId = refId - req.response = resCh - self.newChan <- req - - res := <-resCh - return res.id, res.session, res.responsecode, res.errorstring -} - -func (self *SessionStoreChan) Get(user, id string) (*SessionChan, string, int, string) { - resCh := make(chan getSessionResponse) - req := getSessionRequest{} - req.user = user - req.id = id - req.response = resCh - self.getChan <- req - - res := <-resCh - return res.session, res.refId, res.responsecode, res.errorstring -} - -func (self *SessionStoreChan) List(user, password string, trusted bool) (map[string]string, int, string) { - resCh := make(chan listSessionsResponse) - req := listSessionsRequest{} - req.user = user - req.password = password - req.trusted = trusted - req.response = resCh - self.listChan <- req - - res := <-resCh - return res.sessions, res.responsecode, res.errorstring -} - -func (self *SessionStoreChan) Remove(user, id string) (int, string) { - resCh := make(chan removeSessionResponse) - req := removeSessionRequest{} - req.user = user - req.id = id - req.response = resCh - self.removeChan <- req - - res := <-resCh - return res.responsecode, res.errorstring -} - -func (self *SessionStore) GetInterface() *SessionStoreChan { - ch := &SessionStoreChan{} - ch.newChan = self.newChan - ch.getChan = self.getChan - ch.listChan = self.listChan - ch.removeChan = self.removeChan - return ch -} - -func (self *SessionStore) Cleanup() { - self.quit <- true - <-self.done - close(self.quit) - close(self.done) - close(self.newChan) - close(self.getChan) - close(self.listChan) - close(self.removeChan) -} - -func NewSessionStore(conf *Config, db *rddb.DBChan) (store *SessionStore, err error) { - store = new(SessionStore) - store.conf = conf - store.db = db - store.quit = make(chan bool) - store.done = make(chan bool) - store.store = make(map[string]map[string]*SessionStoreElement) - store.newChan = make(chan newSessionRequest, 10) - store.getChan = make(chan getSessionRequest, 10) - store.listChan = make(chan listSessionsRequest, 10) - store.removeChan = make(chan removeSessionRequest, 10) - - go store.dispatchRequests() - return -} -- cgit v0.10.2