summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Makefile4
-rwxr-xr-xrun-import.py126
-rw-r--r--src/pool-import/dbconfig.go64
-rw-r--r--src/pool-import/main.go137
-rw-r--r--src/pool-import/nulltime.go100
-rw-r--r--src/pool-import/rddb.go228
6 files changed, 12 insertions, 647 deletions
diff --git a/Makefile b/Makefile
index b7e1fa2..27218b6 100644
--- a/Makefile
+++ b/Makefile
@@ -24,9 +24,7 @@ GOCMD := GOPATH=$(curdir) go
EXECUTEABLE := pool-import
-LIBS := "github.com/vaughan0/go-ini" \
- "github.com/ziutek/mymysql/godrv" \
- "code.helsinki.at/rhrd-go/rhimport" \
+LIBS := "code.helsinki.at/rhrd-go/rhimport" \
"code.helsinki.at/rhrd-go/rddb"
diff --git a/run-import.py b/run-import.py
deleted file mode 100755
index 91e7302..0000000
--- a/run-import.py
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/usr/bin/python
-#
-#
-# pool-import
-#
-# Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
-#
-# This file is part of pool-import.
-#
-# pool-import is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# any later version.
-#
-# pool-import is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with pool-import. If not, see <http://www.gnu.org/licenses/>.
-#
-
-'''Radio Helsinki pool-import runner.'''
-
-
-groups = [
- { 'old': "Pool01", 'new': "P_test" },
- { 'old': "Pool02", 'new': "Pbumbumtsc" },
- { 'old': "Pool03", 'new': "Pechochamb" },
- { 'old': "Pool04", 'new': "Pelekthop" },
- { 'old': "Pool05", 'new': "Pblue" },
- { 'old': "Pool06", 'new': "Ppropelles" },
- { 'old': "Pool07", 'new': "Pamon" },
- { 'old': "Pool08", 'new': "Pbrigitte" },
- { 'old': "Pool09", 'new': "Pbrigitten" },
- { 'old': "Pool10", 'new': "Pcine" },
- { 'old': "Pool11", 'new': "Pe5b" },
- { 'old': "Pool12", 'new': "Pelektro" },
- { 'old': "Pool13", 'new': "Plounged" },
- { 'old': "Pool14", 'new': "Plounge" },
- { 'old': "Pool15", 'new': "Pmiles" },
- { 'old': "Pool16", 'new': "Ptomwaits" },
- { 'old': "Pool17", 'new': "Psonne" },
- { 'old': "Pool18", 'new': "Phoefmix1" },
- { 'old': "Pool19", 'new': "Photelpass" },
- { 'old': "Pool20", 'new': "Partcore" },
- { 'old': "Pool21", 'new': "Pbreakcore" },
- { 'old': "Pool22", 'new': "Pbritpop" },
- { 'old': "Pool23", 'new': "Pselchfle" },
- { 'old': "Pool24", 'new': "Ppolanz1" },
- { 'old': "Pool25", 'new': "Pweirdjazz" },
- { 'old': "Pool26", 'new': "Pelesyndub" },
- { 'old': "Pool27", 'new': "Prock" },
- { 'old': "Pool28", 'new': "Pbigbredru" },
- { 'old': "Pool29", 'new': "Pcinleilan" },
- { 'old': "Pool30", 'new': "Pdrone" },
- { 'old': "Pool31", 'new': "PLeichgita" },
- { 'old': "Pool32", 'new': "Pzeitgenoe" },
- { 'old': "Pool33", 'new': "Pelemisch" },
- { 'old': "Pool34", 'new': "Pabunda" },
- { 'old': "Pool35", 'new': "Pska" },
- { 'old': "Pool36", 'new': "Pdemo" },
- { 'old': "Pool37", 'new': "Pgeraeusch" },
- { 'old': "Pool38", 'new': "Pmezopotam" },
- { 'old': "Pool39", 'new': "Pwuggi" },
- { 'old': "Pool40", 'new': "Pkaramba" },
- { 'old': "Pool41", 'new': "Psongbirds" },
- { 'old': "Pool42", 'new': "Pjokebux" },
- { 'old': "Pool43", 'new': "Psingbirds" },
- { 'old': "Pool44", 'new': "Ponconnait" },
- { 'old': "Pool45", 'new': "Pcanzital" },
- { 'old': "Pool46", 'new': "Pmarlies" }
-]
-
-
-def run_one(g):
- import subprocess
- import os
-
- print "start import from %s to %s" % (g['old'], g['new'])
- log = open("%s.log" % (g['old']), "w")
- p = subprocess.Popen(["./pool-import", g['old'], g['new']], stdout=log, stderr=log)
- ret_code = p.wait()
- log.flush()
- log.close()
- print "done importing from %s to %s ... exit_code: %d" % (g['old'], g['new'], ret_code)
-
-if __name__ == '__main__':
- import getopt
- import sys
- from multiprocessing import Pool
-
- usage = '''Radio Helsinki pool-import runner.
-Usage:
- run_import.py [pool-size]
-
-Options:
- -h, --help this help message.
- --pool-size N the number of paralell imports.
-'''
-
- pool_size = 2
-
- try:
- opts, args = getopt.getopt(sys.argv[1:], "h", ["help", "pool-size=" ])
- for o, a in opts:
- if o in ("-h", "--help"):
- print >> sys.stderr, usage
- sys.exit(0)
- elif o == "--pool-size":
- pool_size = int(a)
- else:
- raise getopt.GetoptError('Too many arguments')
-
- if len(args) > 1:
- raise getopt.GetoptError('Too many arguments')
-
- except getopt.GetoptError, msg:
- print >> sys.stderr, "ERROR: %s" % msg
- print >> sys.stderr, usage
- sys.exit(2)
-
-
- p = Pool(pool_size)
- p.map(run_one, groups)
diff --git a/src/pool-import/dbconfig.go b/src/pool-import/dbconfig.go
deleted file mode 100644
index b4733fc..0000000
--- a/src/pool-import/dbconfig.go
+++ /dev/null
@@ -1,64 +0,0 @@
-//
-// pool-import
-//
-// Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
-//
-// This file is part of pool-import.
-//
-// pool-import is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// any later version.
-//
-// pool-import is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with pool-import. If not, see <http://www.gnu.org/licenses/>.
-//
-
-package main
-
-import (
- "github.com/vaughan0/go-ini"
-)
-
-type config struct {
- configfile string
- dbHost string
- dbUser string
- dbPasswd string
- dbDb string
-}
-
-func getIniValue(file ini.File, section string, key string, dflt string) string {
- value, ok := file.Get(section, key)
- if ok {
- return value
- }
- return dflt
-}
-
-func (self *config) readConfigFile() error {
- file, err := ini.LoadFile(self.configfile)
- if err != nil {
- return err
- }
-
- self.dbHost = getIniValue(file, "mySQL", "Hostname", "localhost")
- self.dbUser = getIniValue(file, "mySQL", "Loginname", "rivendell")
- self.dbPasswd = getIniValue(file, "mySQL", "Password", "letmein")
- self.dbDb = getIniValue(file, "mySQL", "Database", "rivendell")
- return nil
-}
-
-func newConfig(configfile string) (conf *config, err error) {
- conf = new(config)
- conf.configfile = configfile
- if err = conf.readConfigFile(); err != nil {
- return
- }
- return
-}
diff --git a/src/pool-import/main.go b/src/pool-import/main.go
index 85dd811..a2c6cdb 100644
--- a/src/pool-import/main.go
+++ b/src/pool-import/main.go
@@ -22,24 +22,17 @@
package main
import (
- "fmt"
"log"
- "net/http"
"os"
"os/signal"
- "sort"
- "strings"
"syscall"
- "time"
- "unicode"
"code.helsinki.at/rhrd-go/rddb"
"code.helsinki.at/rhrd-go/rhimport"
)
const (
- NEW_RD_CONF = "/etc/rd.conf"
- OLD_RD_CONF = "rd.conf"
+ RD_CONF = "/etc/rd.conf"
)
func Done(res rhimport.Result, userdata interface{}) bool {
@@ -48,147 +41,39 @@ func Done(res rhimport.Result, userdata interface{}) bool {
return true
}
-func stripControlCodes(in string) string {
- isC0 := func(r rune) rune {
- if uint32(r) <= unicode.MaxLatin1 && uint8(r) <= 0x1F {
- return -1
- }
- return r
- }
- return strings.Map(isC0, in)
-}
-
-func HandleCart(cart uint, cut uint, artist, album, title, dstgroup string, sessions *rhimport.SessionStore, conf *rhimport.Config, stdlog, dbglog *log.Logger, c <-chan os.Signal) int {
- filename := fmt.Sprintf("%06d_%03d.wav", cart, cut)
-
- stdlog.Printf("")
- stdlog.Printf("********* %s: '%s' / '%s' / '%s'", filename, artist, album, title)
-
- ctx := rhimport.NewContext(conf, nil, stdlog, dbglog)
- ctx.UserName = "importer"
- ctx.Trusted = true
- ctx.GroupName = dstgroup
- ctx.SourceUri = "local:///" + filename
- ctx.FetchConverter = "ffmpeg-bs1770"
- ctx.ExtraMetaData["ARTIST"] = stripControlCodes(artist)
- ctx.ExtraMetaData["ALBUM"] = stripControlCodes(album)
- ctx.ExtraMetaData["TITLE"] = stripControlCodes(title)
-
- _, s, code, errstring := sessions.New(ctx, "")
- if code != http.StatusOK {
- stdlog.Printf(">>>>>>>> ERROR: creating session: %s", errstring)
- return 1
- }
-
- donechan := make(chan rhimport.Result, 1)
- if err := s.AddDoneHandler((chan<- rhimport.Result)(donechan), Done); err != nil {
- stdlog.Printf(">>>>>>> ERROR: adding done handler: %s", err.Error())
- return 1
- }
-
- s.Run(10 * time.Minute)
- var res rhimport.Result
-Loop:
- for {
- select {
- case res = <-donechan:
- break Loop
- case <-c:
- s.Cancel()
- }
- }
- if res.ResponseCode == http.StatusNoContent {
- stdlog.Printf(">>>>>>> CANCELED!!")
- return -1
- }
-
- if res.ResponseCode != http.StatusOK {
- stdlog.Printf(">>>>>>> ERROR: import failed: %s", res.ErrorString)
- return 1
- }
- stdlog.Printf(">>>>>>> SUCCESS: imported into: cart/cut %d/%d", res.Cart, res.Cut)
- return 0
-}
-
func main() {
if len(os.Args) < 3 {
- log.Fatal("Usage: pool-import <old pool group> <new pool group>")
+ log.Fatal("Usage: pool-import <pool group> <directory> [ <directory [ .. ] ]")
}
- oldGroup := os.Args[1]
- newGroup := os.Args[2]
+ group := os.Args[1]
+ directories := os.Args[2:]
- conf, err := rhimport.NewConfig(NEW_RD_CONF, "http://localhost/rd-bin/rdxport.cgi", "/tmp/", "snd/")
+ conf, err := rhimport.NewConfig(RD_CONF, "http://localhost/rd-bin/rdxport.cgi", "/tmp/", "snd/")
if err != nil {
log.Fatal("Error parsing configuration file:", err)
}
- newdb, err := rddb.NewDB(NEW_RD_CONF)
+ db, err := rddb.NewDB(RD_CONF)
if err != nil {
- log.Fatal("Error initializing NEW Rivdenll DB:", err)
+ log.Fatal("Error initializing Rivdenll DB:", err)
}
- defer newdb.Cleanup()
+ defer db.Cleanup()
stdlog := log.New(os.Stderr, "[std] ", log.LstdFlags)
//dbglog := log.New(os.Stderr, "[dbg] ", log.LstdFlags)
var dbglog *log.Logger
- sessions, err := rhimport.NewSessionStore(conf, newdb.GetInterface(), stdlog, dbglog)
+ sessions, err := rhimport.NewSessionStore(conf, db.GetInterface(), stdlog, dbglog)
if err != nil {
log.Fatal("Error initializing Session Store:", err)
}
defer sessions.Cleanup()
- olddb, err := NewDB(OLD_RD_CONF)
- if err != nil {
- log.Fatal("Error initializing OLD Rivdenll DB:", err)
- }
- defer olddb.Cleanup()
- olddbi := olddb.GetInterface()
-
- pool := PoolListEntry{oldGroup, ""}
- carts, err := olddbi.GetPoolCartList(pool)
- if err != nil {
- log.Fatal("Error fetching Pool cart list:", err)
- }
-
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM)
stdlog.Println("***************************************************************")
- stdlog.Printf("*** will import: %d carts from old:%s -> new:%s", len(carts), oldGroup, newGroup)
+ stdlog.Printf("*** will import into group '%s' from %d directories", group, len(directories))
stdlog.Println("***************************************************************")
- var keys []int
- for k := range carts {
- keys = append(keys, int(k))
- }
- sort.Ints(keys)
-
- start := time.Now()
- successCnt := 0
- errCnt := 0
-Loop:
- for _, k := range keys {
- cart := carts[uint(k)]
- if len(cart.Cuts) == 0 {
- stdlog.Printf("Warning: Cart %d has no cuts - ingoring it", cart.Number)
- continue
- } else if len(cart.Cuts) > 1 {
- stdlog.Printf("Warning: Cart %d has multiple cuts - will only use the first", cart.Number)
- }
- cut := cart.Cuts[0].Number
- switch HandleCart(cart.Number, cut, cart.Artist, cart.Album, cart.Title, newGroup, sessions.GetInterface(), conf, stdlog, dbglog, c) {
- case 0:
- successCnt++
- case -1:
- break Loop
- default:
- errCnt++
- }
- }
- stdlog.Println("")
- stdlog.Println("***************************************************************")
- stdlog.Printf("*** %d files imported successfully, %d errors", successCnt, errCnt)
- stdlog.Printf("*** process took %v", time.Since(start))
- stdlog.Println("***************************************************************")
- time.Sleep(time.Second) // give session some time to cleanup
+ <-c
}
diff --git a/src/pool-import/nulltime.go b/src/pool-import/nulltime.go
deleted file mode 100644
index c87d886..0000000
--- a/src/pool-import/nulltime.go
+++ /dev/null
@@ -1,100 +0,0 @@
-//
-// pool-import
-//
-// Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
-//
-// This file is part of pool-import.
-//
-// pool-import is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// any later version.
-//
-// pool-import is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with pool-import. If not, see <http://www.gnu.org/licenses/>.
-//
-
-package main
-
-import (
- "database/sql/driver"
- "fmt"
- "time"
-)
-
-//
-// This is from github.com/go-sql-driver/mysql
-//
-
-const (
- timeFormat = "2006-01-02 15:04:05.999999"
-)
-
-// This NullTime implementation is not driver-specific
-type NullTime struct {
- Time time.Time
- Valid bool // Valid is true if Time is not NULL
-}
-
-// Scan implements the Scanner interface.
-// The value type must be time.Time or string / []byte (formatted time-string),
-// otherwise Scan fails.
-func (nt *NullTime) Scan(value interface{}) (err error) {
- if value == nil {
- nt.Time, nt.Valid = time.Time{}, false
- return
- }
-
- switch v := value.(type) {
- case time.Time:
- nt.Time, nt.Valid = v, true
- return
- case []byte:
- nt.Time, err = parseDateTime(string(v), time.UTC)
- nt.Valid = (err == nil)
- return
- case string:
- nt.Time, err = parseDateTime(v, time.UTC)
- nt.Valid = (err == nil)
- return
- }
-
- nt.Valid = false
- return fmt.Errorf("Can't convert %T to time.Time", value)
-}
-
-// Value implements the driver Valuer interface.
-func (nt NullTime) Value() (driver.Value, error) {
- if !nt.Valid {
- return nil, nil
- }
- return nt.Time, nil
-}
-
-func parseDateTime(str string, loc *time.Location) (t time.Time, err error) {
- base := "0000-00-00 00:00:00.0000000"
- switch len(str) {
- case 10, 19, 21, 22, 23, 24, 25, 26: // up to "YYYY-MM-DD HH:MM:SS.MMMMMM"
- if str == base[:len(str)] {
- return
- }
- t, err = time.Parse(timeFormat[:len(str)], str)
- default:
- err = fmt.Errorf("invalid time string: %s", str)
- return
- }
-
- // Adjust location
- if err == nil && loc != time.UTC {
- y, mo, d := t.Date()
- h, mi, s := t.Clock()
- t, err = time.Date(y, mo, d, h, mi, s, t.Nanosecond(), loc), nil
- }
-
- return
-}
diff --git a/src/pool-import/rddb.go b/src/pool-import/rddb.go
deleted file mode 100644
index 7d2e4c2..0000000
--- a/src/pool-import/rddb.go
+++ /dev/null
@@ -1,228 +0,0 @@
-//
-// pool-import
-//
-// Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
-//
-// This file is part of pool-import.
-//
-// pool-import is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// any later version.
-//
-// pool-import is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with pool-import. If not, see <http://www.gnu.org/licenses/>.
-//
-
-package main
-
-import (
- "database/sql"
- "fmt"
- "regexp"
- "strconv"
- "strings"
- "time"
-
- "github.com/ziutek/mymysql/godrv"
-)
-
-var (
- mysqlTableNameRe = regexp.MustCompile(`^[_0-9a-zA-Z-]+$`)
-)
-
-const (
- DB_VERSION = 182
- defaultCartTitle = "[new cart]"
-)
-
-type CutListEntry struct {
- Number uint
- Evergreen bool
- Description string
- Duration time.Duration
- Imported NullTime
- NumPlayed uint
- LastPlayed NullTime
-}
-
-type CartListEntry struct {
- Number uint
- Exists bool
- Artist string
- Title string
- Album string
- Cuts []CutListEntry
-}
-
-type PoolListEntry struct {
- Group string
- Description string
-}
-
-type getPoolCartListResult struct {
- carts map[uint]CartListEntry
- err error
-}
-
-type getPoolCartListRequest struct {
- pool PoolListEntry
- response chan<- getPoolCartListResult
-}
-
-type db struct {
- conf *config
- dbh *sql.DB
- getPoolCartListChan chan getPoolCartListRequest
- getPoolCartsStmt *sql.Stmt
- quit chan bool
- done chan bool
-}
-
-func (d *db) init() (err error) {
- godrv.Register("SET CHARACTER SET utf8;")
-
- dsn := fmt.Sprintf("tcp:%s:3306*%s/%s/%s", d.conf.dbHost, d.conf.dbDb, d.conf.dbUser, d.conf.dbPasswd)
- if d.dbh, err = sql.Open("mymysql", dsn); err != nil {
- return
- }
-
- var dbver int
- err = d.dbh.QueryRow("select DB from VERSION;").Scan(&dbver)
- if err != nil {
- err = fmt.Errorf("fetching version: %s", err)
- return
- }
- if dbver != DB_VERSION {
- err = fmt.Errorf("version mismatch is %d, should be %d", dbver, DB_VERSION)
- return
- }
-
- if d.getPoolCartsStmt, err = d.dbh.Prepare("select CART.NUMBER,CART.ARTIST,CART.TITLE,CART.ALBUM,CUTS.CUT_NAME,CUTS.EVERGREEN,CUTS.DESCRIPTION,CUTS.LENGTH,CUTS.ORIGIN_DATETIME,CUTS.PLAY_COUNTER,CUTS.LAST_PLAY_DATETIME from CUTS,CART where CUTS.CART_NUMBER = CART.NUMBER and CART.GROUP_NAME = ?;"); err != nil {
- return
- }
-
- return
-}
-
-func (d *db) getPoolCartList(pool PoolListEntry) (result getPoolCartListResult) {
- var rows *sql.Rows
- if rows, result.err = d.getPoolCartsStmt.Query(pool.Group); result.err != nil {
- return
- }
- defer rows.Close()
-
- result.carts = make(map[uint]CartListEntry)
- for rows.Next() {
- var cut CutListEntry
- var cart, length uint
- var cutName, evergreen string
- var artist, title, album sql.NullString
-
- if result.err = rows.Scan(&cart, &artist, &title, &album, &cutName, &evergreen, &cut.Description, &length, &cut.Imported, &cut.NumPlayed, &cut.LastPlayed); result.err != nil {
- return
- }
-
- parts := strings.Split(cutName, "_")
- if len(parts) == 2 {
- if cn, converr := strconv.ParseUint(parts[1], 10, 32); converr != nil {
- continue
- } else {
- cut.Number = uint(cn)
- }
- } else {
- continue
- }
- switch evergreen {
- case "Y":
- cut.Evergreen = true
- default:
- cut.Evergreen = false
- }
- cut.Duration = time.Duration(length) * time.Millisecond
-
- if c, exists := result.carts[cart]; !exists {
- c = CartListEntry{cart, true, artist.String, title.String, album.String, nil}
- c.Cuts = append(c.Cuts, cut)
- result.carts[cart] = c
- } else {
- c.Cuts = append(c.Cuts, cut)
- }
- }
- result.err = rows.Err()
- return
-}
-
-func (d *db) dispatchRequests() {
- defer func() { d.done <- true }()
- for {
- select {
- case <-d.quit:
- return
- case req := <-d.getPoolCartListChan:
- req.response <- d.getPoolCartList(req.pool)
- }
- }
-}
-
-// *********************************************************
-// Public Interface
-
-type DB struct {
- getPoolCartListChan chan<- getPoolCartListRequest
-}
-
-func (d *DB) GetPoolCartList(pool PoolListEntry) (map[uint]CartListEntry, error) {
- resCh := make(chan getPoolCartListResult)
- req := getPoolCartListRequest{}
- req.pool = pool
- req.response = resCh
- d.getPoolCartListChan <- req
-
- res := <-resCh
- if res.err != nil {
- return nil, res.err
- }
- return res.carts, nil
-}
-
-func (d *db) GetInterface() *DB {
- ch := &DB{}
- ch.getPoolCartListChan = d.getPoolCartListChan
- return ch
-}
-
-func (d *db) Cleanup() {
- d.quit <- true
- <-d.done
- close(d.quit)
- close(d.done)
- if d.dbh != nil {
- d.dbh.Close()
- }
- if d.getPoolCartsStmt != nil {
- d.getPoolCartsStmt.Close()
- }
-}
-
-func NewDB(configfile string) (d *db, err error) {
- d = new(db)
- if d.conf, err = newConfig(configfile); err != nil {
- return
- }
- d.quit = make(chan bool)
- d.done = make(chan bool)
- d.getPoolCartListChan = make(chan getPoolCartListRequest, 10)
-
- if err = d.init(); err != nil {
- return
- }
-
- go d.dispatchRequests()
- return
-}