heartbeat handling works now

3 files changed, 66 insertions, 22 deletions

diff --git a/src/rhctl/audio_switch.go b/src/rhctl/audio_switch.go
index 537e108..2e0a965 100644
--- a/src/rhctl/audio_switch.go
+++ b/src/rhctl/audio_switch.go
@@ -143,7 +143,10 @@ func (sw *AudioSwitch) Run() {
 
 func SwitchInit(conf *Config) (sw *AudioSwitch, err error) {
 	sw = &AudioSwitch{}
-	sw.timeout = conf.Audioswitch.Timeout.Duration
+	sw.timeout = time.Second
+	if conf.Audioswitch.Timeout.Duration > time.Duration(0) {
+		sw.timeout = conf.Audioswitch.Timeout.Duration
+	}
 	sw.Commands = make(chan *SwitchCommand, 8)
 	sw.Updates = make(chan SwitchUpdate, 32)
 
diff --git a/src/rhctl/conf.go b/src/rhctl/conf.go
index 7a1aea0..0203471 100644
--- a/src/rhctl/conf.go
+++ b/src/rhctl/conf.go
@@ -37,10 +37,12 @@ type Config struct {
 	}
 
 	Servers map[string]struct {
-		ControlDevice     string   `toml:"ctrl_dev"`
-		ControlBaudrate   Baudrate `toml:"ctrl_baud"`
-		HeartbeatDevice   string   `toml:"heartbeat_dev"`
-		HeartbeatBaudrate Baudrate `toml:"heartbeat_baud"`
+		ControlDevice      string   `toml:"ctrl_dev"`
+		ControlBaudrate    Baudrate `toml:"ctrl_baud"`
+		HeartbeatDevice    string   `toml:"heartbeat_dev"`
+		HeartbeatBaudrate  Baudrate `toml:"heartbeat_baud"`
+		HeartbeatTimeout   Duration `toml:"heartbeat_timeout"`
+		HeartbeatThreshold uint     `toml:"heartbeat_threshold"`
 	}
 
 	Clients struct {
@@ -73,10 +75,5 @@ func ReadConfig(configfile string) (conf *Config, err error) {
 	if err = decoder.Decode(conf); err != nil {
 		return
 	}
-
-	if conf.Audioswitch.Timeout.Duration <= time.Duration(0) {
-		conf.Audioswitch.Timeout.Duration = time.Second
-	}
-
 	return
 }
diff --git a/src/rhctl/playout_server.go b/src/rhctl/playout_server.go
index 506279b..5666586 100644
--- a/src/rhctl/playout_server.go
+++ b/src/rhctl/playout_server.go
@@ -24,26 +24,41 @@ package main
 import (
 	"fmt"
 	"strings"
+	"time"
 )
 
 type ServerHealth bool
 
 const (
-	Dead  ServerHealth = false
-	Alive              = true
+	ServerDead  ServerHealth = false
+	ServerAlive ServerHealth = true
 )
 
+func (s ServerHealth) String() string {
+	switch s {
+	case ServerDead:
+		return "dead"
+	case ServerAlive:
+		return "alive"
+	}
+	return "unknown"
+}
+
 type ServerStatus struct {
-	health  ServerHealth
-	channel string
+	Health  ServerHealth
+	Channel string
 }
 
 type PlayoutServer struct {
-	name      string
-	control   *SerialPort
-	heartbeat *SerialPort
-	status    ServerStatus
-	Updates   chan ServerStatus
+	name        string
+	control     *SerialPort
+	heartbeat   *SerialPort
+	hbtimeout   time.Duration
+	hbtimer     *time.Timer
+	hbthreshold uint
+	hbcnt       uint
+	status      ServerStatus
+	Updates     chan ServerStatus
 }
 
 func (srv *PlayoutServer) handleControl(data string) {
@@ -54,7 +69,7 @@ func (srv *PlayoutServer) handleControl(data string) {
 			rhl.Printf("Server(%s) sent empty channel name", srv.name)
 			return
 		}
-		srv.status.channel = data[8:]
+		srv.status.Channel = data[8:]
 		srv.Updates <- srv.status
 		return
 	}
@@ -63,12 +78,31 @@ func (srv *PlayoutServer) handleControl(data string) {
 
 func (srv *PlayoutServer) handleHeartbeat(data string) {
 	rhdl.Printf("Server(%s): got heartbeat message: %q", srv.name, data)
+	srv.hbtimer.Reset(srv.hbtimeout)
+	old := srv.status.Health
+	srv.hbcnt++
+	if srv.hbcnt < srv.hbthreshold {
+		return
+	}
+	srv.status.Health = ServerAlive
+	if old != srv.status.Health {
+		srv.Updates <- srv.status
+	}
+}
+
+func (srv *PlayoutServer) handleHBTimeout() {
+	rhl.Printf("Server(%s): heartbeat timed-out", srv.name)
+	srv.hbcnt = 0
+	srv.status.Health = ServerDead
+	srv.Updates <- srv.status
 }
 
 func (srv *PlayoutServer) Run() {
 	stop := make(chan bool)
 	srv.control.Run(stop)
 	srv.heartbeat.Run(stop)
+	srv.hbtimer = time.NewTimer(srv.hbtimeout)
+	srv.hbcnt = 0
 
 	rhdl.Printf("Server(%s) handler running...", srv.name)
 	for {
@@ -79,6 +113,8 @@ func (srv *PlayoutServer) Run() {
 			srv.handleControl(data)
 		case data := <-srv.heartbeat.rx:
 			srv.handleHeartbeat(data)
+		case <-srv.hbtimer.C:
+			srv.handleHBTimeout()
 		}
 	}
 }
@@ -86,8 +122,16 @@ func (srv *PlayoutServer) Run() {
 func ServerInit(name string, conf *Config) (srv *PlayoutServer, err error) {
 	srv = &PlayoutServer{}
 	srv.name = name
-	srv.status.health = Dead
-	srv.status.channel = ""
+	srv.hbtimeout = 3 * time.Second
+	if conf.Servers[name].HeartbeatTimeout.Duration > time.Duration(0) {
+		srv.hbtimeout = conf.Servers[name].HeartbeatTimeout.Duration
+	}
+	srv.hbthreshold = 10
+	if conf.Servers[name].HeartbeatThreshold > 0 {
+		srv.hbthreshold = conf.Servers[name].HeartbeatThreshold
+	}
+	srv.status.Health = ServerDead
+	srv.status.Channel = ""
 	srv.Updates = make(chan ServerStatus, 8)
 
 	if srv.control, err = SerialOpen(conf.Servers[name].ControlDevice, conf.Servers[name].ControlBaudrate, "\r\n"); err != nil {