From 972c2951d71bbb2e25e1d5aa8197921ada1997a1 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Wed, 28 Dec 2016 23:32:59 +0100 Subject: removed legacy pool-importer diff --git a/Makefile b/Makefile index b7e1fa2..27218b6 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,7 @@ GOCMD := GOPATH=$(curdir) go EXECUTEABLE := pool-import -LIBS := "github.com/vaughan0/go-ini" \ - "github.com/ziutek/mymysql/godrv" \ - "code.helsinki.at/rhrd-go/rhimport" \ +LIBS := "code.helsinki.at/rhrd-go/rhimport" \ "code.helsinki.at/rhrd-go/rddb" diff --git a/run-import.py b/run-import.py deleted file mode 100755 index 91e7302..0000000 --- a/run-import.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/python -# -# -# pool-import -# -# Copyright (C) 2016 Christian Pointner -# -# This file is part of pool-import. -# -# pool-import is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# pool-import is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with pool-import. If not, see . -# - -'''Radio Helsinki pool-import runner.''' - - -groups = [ - { 'old': "Pool01", 'new': "P_test" }, - { 'old': "Pool02", 'new': "Pbumbumtsc" }, - { 'old': "Pool03", 'new': "Pechochamb" }, - { 'old': "Pool04", 'new': "Pelekthop" }, - { 'old': "Pool05", 'new': "Pblue" }, - { 'old': "Pool06", 'new': "Ppropelles" }, - { 'old': "Pool07", 'new': "Pamon" }, - { 'old': "Pool08", 'new': "Pbrigitte" }, - { 'old': "Pool09", 'new': "Pbrigitten" }, - { 'old': "Pool10", 'new': "Pcine" }, - { 'old': "Pool11", 'new': "Pe5b" }, - { 'old': "Pool12", 'new': "Pelektro" }, - { 'old': "Pool13", 'new': "Plounged" }, - { 'old': "Pool14", 'new': "Plounge" }, - { 'old': "Pool15", 'new': "Pmiles" }, - { 'old': "Pool16", 'new': "Ptomwaits" }, - { 'old': "Pool17", 'new': "Psonne" }, - { 'old': "Pool18", 'new': "Phoefmix1" }, - { 'old': "Pool19", 'new': "Photelpass" }, - { 'old': "Pool20", 'new': "Partcore" }, - { 'old': "Pool21", 'new': "Pbreakcore" }, - { 'old': "Pool22", 'new': "Pbritpop" }, - { 'old': "Pool23", 'new': "Pselchfle" }, - { 'old': "Pool24", 'new': "Ppolanz1" }, - { 'old': "Pool25", 'new': "Pweirdjazz" }, - { 'old': "Pool26", 'new': "Pelesyndub" }, - { 'old': "Pool27", 'new': "Prock" }, - { 'old': "Pool28", 'new': "Pbigbredru" }, - { 'old': "Pool29", 'new': "Pcinleilan" }, - { 'old': "Pool30", 'new': "Pdrone" }, - { 'old': "Pool31", 'new': "PLeichgita" }, - { 'old': "Pool32", 'new': "Pzeitgenoe" }, - { 'old': "Pool33", 'new': "Pelemisch" }, - { 'old': "Pool34", 'new': "Pabunda" }, - { 'old': "Pool35", 'new': "Pska" }, - { 'old': "Pool36", 'new': "Pdemo" }, - { 'old': "Pool37", 'new': "Pgeraeusch" }, - { 'old': "Pool38", 'new': "Pmezopotam" }, - { 'old': "Pool39", 'new': "Pwuggi" }, - { 'old': "Pool40", 'new': "Pkaramba" }, - { 'old': "Pool41", 'new': "Psongbirds" }, - { 'old': "Pool42", 'new': "Pjokebux" }, - { 'old': "Pool43", 'new': "Psingbirds" }, - { 'old': "Pool44", 'new': "Ponconnait" }, - { 'old': "Pool45", 'new': "Pcanzital" }, - { 'old': "Pool46", 'new': "Pmarlies" } -] - - -def run_one(g): - import subprocess - import os - - print "start import from %s to %s" % (g['old'], g['new']) - log = open("%s.log" % (g['old']), "w") - p = subprocess.Popen(["./pool-import", g['old'], g['new']], stdout=log, stderr=log) - ret_code = p.wait() - log.flush() - log.close() - print "done importing from %s to %s ... exit_code: %d" % (g['old'], g['new'], ret_code) - -if __name__ == '__main__': - import getopt - import sys - from multiprocessing import Pool - - usage = '''Radio Helsinki pool-import runner. -Usage: - run_import.py [pool-size] - -Options: - -h, --help this help message. - --pool-size N the number of paralell imports. -''' - - pool_size = 2 - - try: - opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "pool-size=" ]) - for o, a in opts: - if o in ("-h", "--help"): - print >> sys.stderr, usage - sys.exit(0) - elif o == "--pool-size": - pool_size = int(a) - else: - raise getopt.GetoptError('Too many arguments') - - if len(args) > 1: - raise getopt.GetoptError('Too many arguments') - - except getopt.GetoptError, msg: - print >> sys.stderr, "ERROR: %s" % msg - print >> sys.stderr, usage - sys.exit(2) - - - p = Pool(pool_size) - p.map(run_one, groups) diff --git a/src/pool-import/dbconfig.go b/src/pool-import/dbconfig.go deleted file mode 100644 index b4733fc..0000000 --- a/src/pool-import/dbconfig.go +++ /dev/null @@ -1,64 +0,0 @@ -// -// pool-import -// -// Copyright (C) 2016 Christian Pointner -// -// This file is part of pool-import. -// -// pool-import is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// pool-import is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with pool-import. If not, see . -// - -package main - -import ( - "github.com/vaughan0/go-ini" -) - -type config struct { - configfile string - dbHost string - dbUser string - dbPasswd string - dbDb string -} - -func getIniValue(file ini.File, section string, key string, dflt string) string { - value, ok := file.Get(section, key) - if ok { - return value - } - return dflt -} - -func (self *config) readConfigFile() error { - file, err := ini.LoadFile(self.configfile) - if err != nil { - return err - } - - self.dbHost = getIniValue(file, "mySQL", "Hostname", "localhost") - self.dbUser = getIniValue(file, "mySQL", "Loginname", "rivendell") - self.dbPasswd = getIniValue(file, "mySQL", "Password", "letmein") - self.dbDb = getIniValue(file, "mySQL", "Database", "rivendell") - return nil -} - -func newConfig(configfile string) (conf *config, err error) { - conf = new(config) - conf.configfile = configfile - if err = conf.readConfigFile(); err != nil { - return - } - return -} diff --git a/src/pool-import/main.go b/src/pool-import/main.go index 85dd811..a2c6cdb 100644 --- a/src/pool-import/main.go +++ b/src/pool-import/main.go @@ -22,24 +22,17 @@ package main import ( - "fmt" "log" - "net/http" "os" "os/signal" - "sort" - "strings" "syscall" - "time" - "unicode" "code.helsinki.at/rhrd-go/rddb" "code.helsinki.at/rhrd-go/rhimport" ) const ( - NEW_RD_CONF = "/etc/rd.conf" - OLD_RD_CONF = "rd.conf" + RD_CONF = "/etc/rd.conf" ) func Done(res rhimport.Result, userdata interface{}) bool { @@ -48,147 +41,39 @@ func Done(res rhimport.Result, userdata interface{}) bool { return true } -func stripControlCodes(in string) string { - isC0 := func(r rune) rune { - if uint32(r) <= unicode.MaxLatin1 && uint8(r) <= 0x1F { - return -1 - } - return r - } - return strings.Map(isC0, in) -} - -func HandleCart(cart uint, cut uint, artist, album, title, dstgroup string, sessions *rhimport.SessionStore, conf *rhimport.Config, stdlog, dbglog *log.Logger, c <-chan os.Signal) int { - filename := fmt.Sprintf("%06d_%03d.wav", cart, cut) - - stdlog.Printf("") - stdlog.Printf("********* %s: '%s' / '%s' / '%s'", filename, artist, album, title) - - ctx := rhimport.NewContext(conf, nil, stdlog, dbglog) - ctx.UserName = "importer" - ctx.Trusted = true - ctx.GroupName = dstgroup - ctx.SourceUri = "local:///" + filename - ctx.FetchConverter = "ffmpeg-bs1770" - ctx.ExtraMetaData["ARTIST"] = stripControlCodes(artist) - ctx.ExtraMetaData["ALBUM"] = stripControlCodes(album) - ctx.ExtraMetaData["TITLE"] = stripControlCodes(title) - - _, s, code, errstring := sessions.New(ctx, "") - if code != http.StatusOK { - stdlog.Printf(">>>>>>>> ERROR: creating session: %s", errstring) - return 1 - } - - donechan := make(chan rhimport.Result, 1) - if err := s.AddDoneHandler((chan<- rhimport.Result)(donechan), Done); err != nil { - stdlog.Printf(">>>>>>> ERROR: adding done handler: %s", err.Error()) - return 1 - } - - s.Run(10 * time.Minute) - var res rhimport.Result -Loop: - for { - select { - case res = <-donechan: - break Loop - case <-c: - s.Cancel() - } - } - if res.ResponseCode == http.StatusNoContent { - stdlog.Printf(">>>>>>> CANCELED!!") - return -1 - } - - if res.ResponseCode != http.StatusOK { - stdlog.Printf(">>>>>>> ERROR: import failed: %s", res.ErrorString) - return 1 - } - stdlog.Printf(">>>>>>> SUCCESS: imported into: cart/cut %d/%d", res.Cart, res.Cut) - return 0 -} - func main() { if len(os.Args) < 3 { - log.Fatal("Usage: pool-import ") + log.Fatal("Usage: pool-import [ new:%s", len(carts), oldGroup, newGroup) + stdlog.Printf("*** will import into group '%s' from %d directories", group, len(directories)) stdlog.Println("***************************************************************") - var keys []int - for k := range carts { - keys = append(keys, int(k)) - } - sort.Ints(keys) - - start := time.Now() - successCnt := 0 - errCnt := 0 -Loop: - for _, k := range keys { - cart := carts[uint(k)] - if len(cart.Cuts) == 0 { - stdlog.Printf("Warning: Cart %d has no cuts - ingoring it", cart.Number) - continue - } else if len(cart.Cuts) > 1 { - stdlog.Printf("Warning: Cart %d has multiple cuts - will only use the first", cart.Number) - } - cut := cart.Cuts[0].Number - switch HandleCart(cart.Number, cut, cart.Artist, cart.Album, cart.Title, newGroup, sessions.GetInterface(), conf, stdlog, dbglog, c) { - case 0: - successCnt++ - case -1: - break Loop - default: - errCnt++ - } - } - stdlog.Println("") - stdlog.Println("***************************************************************") - stdlog.Printf("*** %d files imported successfully, %d errors", successCnt, errCnt) - stdlog.Printf("*** process took %v", time.Since(start)) - stdlog.Println("***************************************************************") - time.Sleep(time.Second) // give session some time to cleanup + <-c } diff --git a/src/pool-import/nulltime.go b/src/pool-import/nulltime.go deleted file mode 100644 index c87d886..0000000 --- a/src/pool-import/nulltime.go +++ /dev/null @@ -1,100 +0,0 @@ -// -// pool-import -// -// Copyright (C) 2016 Christian Pointner -// -// This file is part of pool-import. -// -// pool-import is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// pool-import is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with pool-import. If not, see . -// - -package main - -import ( - "database/sql/driver" - "fmt" - "time" -) - -// -// This is from github.com/go-sql-driver/mysql -// - -const ( - timeFormat = "2006-01-02 15:04:05.999999" -) - -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - -func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { - base := "0000-00-00 00:00:00.0000000" - switch len(str) { - case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM" - if str == base[:len(str)] { - return - } - t, err = time.Parse(timeFormat[:len(str)], str) - default: - err = fmt.Errorf("invalid time string: %s", str) - return - } - - // Adjust location - if err == nil && loc != time.UTC { - y, mo, d := t.Date() - h, mi, s := t.Clock() - t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil - } - - return -} diff --git a/src/pool-import/rddb.go b/src/pool-import/rddb.go deleted file mode 100644 index 7d2e4c2..0000000 --- a/src/pool-import/rddb.go +++ /dev/null @@ -1,228 +0,0 @@ -// -// pool-import -// -// Copyright (C) 2016 Christian Pointner -// -// This file is part of pool-import. -// -// pool-import is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// any later version. -// -// pool-import is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with pool-import. If not, see . -// - -package main - -import ( - "database/sql" - "fmt" - "regexp" - "strconv" - "strings" - "time" - - "github.com/ziutek/mymysql/godrv" -) - -var ( - mysqlTableNameRe = regexp.MustCompile(`^[_0-9a-zA-Z-]+$`) -) - -const ( - DB_VERSION = 182 - defaultCartTitle = "[new cart]" -) - -type CutListEntry struct { - Number uint - Evergreen bool - Description string - Duration time.Duration - Imported NullTime - NumPlayed uint - LastPlayed NullTime -} - -type CartListEntry struct { - Number uint - Exists bool - Artist string - Title string - Album string - Cuts []CutListEntry -} - -type PoolListEntry struct { - Group string - Description string -} - -type getPoolCartListResult struct { - carts map[uint]CartListEntry - err error -} - -type getPoolCartListRequest struct { - pool PoolListEntry - response chan<- getPoolCartListResult -} - -type db struct { - conf *config - dbh *sql.DB - getPoolCartListChan chan getPoolCartListRequest - getPoolCartsStmt *sql.Stmt - quit chan bool - done chan bool -} - -func (d *db) init() (err error) { - godrv.Register("SET CHARACTER SET utf8;") - - dsn := fmt.Sprintf("tcp:%s:3306*%s/%s/%s", d.conf.dbHost, d.conf.dbDb, d.conf.dbUser, d.conf.dbPasswd) - if d.dbh, err = sql.Open("mymysql", dsn); err != nil { - return - } - - var dbver int - err = d.dbh.QueryRow("select DB from VERSION;").Scan(&dbver) - if err != nil { - err = fmt.Errorf("fetching version: %s", err) - return - } - if dbver != DB_VERSION { - err = fmt.Errorf("version mismatch is %d, should be %d", dbver, DB_VERSION) - return - } - - if d.getPoolCartsStmt, err = d.dbh.Prepare("select CART.NUMBER,CART.ARTIST,CART.TITLE,CART.ALBUM,CUTS.CUT_NAME,CUTS.EVERGREEN,CUTS.DESCRIPTION,CUTS.LENGTH,CUTS.ORIGIN_DATETIME,CUTS.PLAY_COUNTER,CUTS.LAST_PLAY_DATETIME from CUTS,CART where CUTS.CART_NUMBER = CART.NUMBER and CART.GROUP_NAME = ?;"); err != nil { - return - } - - return -} - -func (d *db) getPoolCartList(pool PoolListEntry) (result getPoolCartListResult) { - var rows *sql.Rows - if rows, result.err = d.getPoolCartsStmt.Query(pool.Group); result.err != nil { - return - } - defer rows.Close() - - result.carts = make(map[uint]CartListEntry) - for rows.Next() { - var cut CutListEntry - var cart, length uint - var cutName, evergreen string - var artist, title, album sql.NullString - - if result.err = rows.Scan(&cart, &artist, &title, &album, &cutName, &evergreen, &cut.Description, &length, &cut.Imported, &cut.NumPlayed, &cut.LastPlayed); result.err != nil { - return - } - - parts := strings.Split(cutName, "_") - if len(parts) == 2 { - if cn, converr := strconv.ParseUint(parts[1], 10, 32); converr != nil { - continue - } else { - cut.Number = uint(cn) - } - } else { - continue - } - switch evergreen { - case "Y": - cut.Evergreen = true - default: - cut.Evergreen = false - } - cut.Duration = time.Duration(length) * time.Millisecond - - if c, exists := result.carts[cart]; !exists { - c = CartListEntry{cart, true, artist.String, title.String, album.String, nil} - c.Cuts = append(c.Cuts, cut) - result.carts[cart] = c - } else { - c.Cuts = append(c.Cuts, cut) - } - } - result.err = rows.Err() - return -} - -func (d *db) dispatchRequests() { - defer func() { d.done <- true }() - for { - select { - case <-d.quit: - return - case req := <-d.getPoolCartListChan: - req.response <- d.getPoolCartList(req.pool) - } - } -} - -// ********************************************************* -// Public Interface - -type DB struct { - getPoolCartListChan chan<- getPoolCartListRequest -} - -func (d *DB) GetPoolCartList(pool PoolListEntry) (map[uint]CartListEntry, error) { - resCh := make(chan getPoolCartListResult) - req := getPoolCartListRequest{} - req.pool = pool - req.response = resCh - d.getPoolCartListChan <- req - - res := <-resCh - if res.err != nil { - return nil, res.err - } - return res.carts, nil -} - -func (d *db) GetInterface() *DB { - ch := &DB{} - ch.getPoolCartListChan = d.getPoolCartListChan - return ch -} - -func (d *db) Cleanup() { - d.quit <- true - <-d.done - close(d.quit) - close(d.done) - if d.dbh != nil { - d.dbh.Close() - } - if d.getPoolCartsStmt != nil { - d.getPoolCartsStmt.Close() - } -} - -func NewDB(configfile string) (d *db, err error) { - d = new(db) - if d.conf, err = newConfig(configfile); err != nil { - return - } - d.quit = make(chan bool) - d.done = make(chan bool) - d.getPoolCartListChan = make(chan getPoolCartListRequest, 10) - - if err = d.init(); err != nil { - return - } - - go d.dispatchRequests() - return -} -- cgit v0.10.2