cmd/puppeth: concurrent server dials and health checks

parent 8c78449a
...@@ -28,6 +28,7 @@ import ( ...@@ -28,6 +28,7 @@ import (
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
"sync"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
...@@ -75,7 +76,8 @@ type wizard struct { ...@@ -75,7 +76,8 @@ type wizard struct {
servers map[string]*sshClient // SSH connections to servers to administer servers map[string]*sshClient // SSH connections to servers to administer
services map[string][]string // Ethereum services known to be running on servers services map[string][]string // Ethereum services known to be running on servers
in *bufio.Reader // Wrapper around stdin to allow reading user input in *bufio.Reader // Wrapper around stdin to allow reading user input
lock sync.Mutex // Lock to protect configs during concurrent service discovery
} }
// read reads a single line from stdin, trimming if from spaces. // read reads a single line from stdin, trimming if from spaces.
......
...@@ -24,6 +24,7 @@ import ( ...@@ -24,6 +24,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
) )
...@@ -80,14 +81,25 @@ func (w *wizard) run() { ...@@ -80,14 +81,25 @@ func (w *wizard) run() {
} else if err := json.Unmarshal(blob, &w.conf); err != nil { } else if err := json.Unmarshal(blob, &w.conf); err != nil {
log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err) log.Crit("Previous configuration corrupted", "path", w.conf.path, "err", err)
} else { } else {
// Dial all previously known servers concurrently
var pend sync.WaitGroup
for server, pubkey := range w.conf.Servers { for server, pubkey := range w.conf.Servers {
log.Info("Dialing previously configured server", "server", server) pend.Add(1)
client, err := dial(server, pubkey)
if err != nil { go func(server string, pubkey []byte) {
log.Error("Previous server unreachable", "server", server, "err", err) defer pend.Done()
}
w.servers[server] = client log.Info("Dialing previously configured server", "server", server)
client, err := dial(server, pubkey)
if err != nil {
log.Error("Previous server unreachable", "server", server, "err", err)
}
w.lock.Lock()
w.servers[server] = client
w.lock.Unlock()
}(server, pubkey)
} }
pend.Wait()
w.networkStats() w.networkStats()
} }
// Basics done, loop ad infinitum about what to do // Basics done, loop ad infinitum about what to do
......
...@@ -21,6 +21,7 @@ import ( ...@@ -21,6 +21,7 @@ import (
"os" "os"
"sort" "sort"
"strings" "strings"
"sync"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
...@@ -34,112 +35,143 @@ func (w *wizard) networkStats() { ...@@ -34,112 +35,143 @@ func (w *wizard) networkStats() {
log.Error("No remote machines to gather stats from") log.Error("No remote machines to gather stats from")
return return
} }
protips := new(protips) // Clear out some previous configs to refill from current scan
w.conf.ethstats = ""
w.conf.bootFull = w.conf.bootFull[:0]
w.conf.bootLight = w.conf.bootLight[:0]
// Iterate over all the specified hosts and check their status // Iterate over all the specified hosts and check their status
stats := make(serverStats) var pend sync.WaitGroup
stats := make(serverStats)
for server, pubkey := range w.conf.Servers { for server, pubkey := range w.conf.Servers {
client := w.servers[server] pend.Add(1)
logger := log.New("server", server)
logger.Info("Starting remote server health-check") // Gather the service stats for each server concurrently
go func(server string, pubkey []byte) {
stat := &serverStat{ defer pend.Done()
address: client.address,
services: make(map[string]map[string]string), stat := w.gatherStats(server, pubkey, w.servers[server])
}
stats[client.server] = stat // All status checks complete, report and check next server
w.lock.Lock()
if client == nil { defer w.lock.Unlock()
conn, err := dial(server, pubkey)
if err != nil { delete(w.services, server)
logger.Error("Failed to establish remote connection", "err", err) for service := range stat.services {
stat.failure = err.Error() w.services[server] = append(w.services[server], service)
continue
} }
client = conn stats[server] = stat
}(server, pubkey)
}
pend.Wait()
// Print any collected stats and return
stats.render()
}
// gatherStats gathers service statistics for a particular remote server.
func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *serverStat {
// Gather some global stats to feed into the wizard
var (
genesis string
ethstats string
bootFull []string
bootLight []string
)
// Ensure a valid SSH connection to the remote server
logger := log.New("server", server)
logger.Info("Starting remote server health-check")
stat := &serverStat{
address: client.address,
services: make(map[string]map[string]string),
}
if client == nil {
conn, err := dial(server, pubkey)
if err != nil {
logger.Error("Failed to establish remote connection", "err", err)
stat.failure = err.Error()
return stat
} }
// Client connected one way or another, run health-checks client = conn
logger.Debug("Checking for nginx availability") }
if infos, err := checkNginx(client, w.network); err != nil { // Client connected one way or another, run health-checks
if err != ErrServiceUnknown { logger.Debug("Checking for nginx availability")
stat.services["nginx"] = map[string]string{"offline": err.Error()} if infos, err := checkNginx(client, w.network); err != nil {
} if err != ErrServiceUnknown {
} else { stat.services["nginx"] = map[string]string{"offline": err.Error()}
stat.services["nginx"] = infos.Report()
} }
logger.Debug("Checking for ethstats availability") } else {
if infos, err := checkEthstats(client, w.network); err != nil { stat.services["nginx"] = infos.Report()
if err != ErrServiceUnknown { }
stat.services["ethstats"] = map[string]string{"offline": err.Error()} logger.Debug("Checking for ethstats availability")
} if infos, err := checkEthstats(client, w.network); err != nil {
} else { if err != ErrServiceUnknown {
stat.services["ethstats"] = infos.Report() stat.services["ethstats"] = map[string]string{"offline": err.Error()}
protips.ethstats = infos.config
} }
logger.Debug("Checking for bootnode availability") } else {
if infos, err := checkNode(client, w.network, true); err != nil { stat.services["ethstats"] = infos.Report()
if err != ErrServiceUnknown { ethstats = infos.config
stat.services["bootnode"] = map[string]string{"offline": err.Error()} }
} logger.Debug("Checking for bootnode availability")
} else { if infos, err := checkNode(client, w.network, true); err != nil {
stat.services["bootnode"] = infos.Report() if err != ErrServiceUnknown {
stat.services["bootnode"] = map[string]string{"offline": err.Error()}
protips.genesis = string(infos.genesis)
protips.bootFull = append(protips.bootFull, infos.enodeFull)
if infos.enodeLight != "" {
protips.bootLight = append(protips.bootLight, infos.enodeLight)
}
} }
logger.Debug("Checking for sealnode availability") } else {
if infos, err := checkNode(client, w.network, false); err != nil { stat.services["bootnode"] = infos.Report()
if err != ErrServiceUnknown {
stat.services["sealnode"] = map[string]string{"offline": err.Error()} genesis = string(infos.genesis)
} bootFull = append(bootFull, infos.enodeFull)
} else { if infos.enodeLight != "" {
stat.services["sealnode"] = infos.Report() bootLight = append(bootLight, infos.enodeLight)
protips.genesis = string(infos.genesis)
} }
logger.Debug("Checking for faucet availability") }
if infos, err := checkFaucet(client, w.network); err != nil { logger.Debug("Checking for sealnode availability")
if err != ErrServiceUnknown { if infos, err := checkNode(client, w.network, false); err != nil {
stat.services["faucet"] = map[string]string{"offline": err.Error()} if err != ErrServiceUnknown {
} stat.services["sealnode"] = map[string]string{"offline": err.Error()}
} else {
stat.services["faucet"] = infos.Report()
} }
logger.Debug("Checking for dashboard availability") } else {
if infos, err := checkDashboard(client, w.network); err != nil { stat.services["sealnode"] = infos.Report()
if err != ErrServiceUnknown { genesis = string(infos.genesis)
stat.services["dashboard"] = map[string]string{"offline": err.Error()} }
} logger.Debug("Checking for faucet availability")
} else { if infos, err := checkFaucet(client, w.network); err != nil {
stat.services["dashboard"] = infos.Report() if err != ErrServiceUnknown {
stat.services["faucet"] = map[string]string{"offline": err.Error()}
} }
// All status checks complete, report and check next server } else {
delete(w.services, server) stat.services["faucet"] = infos.Report()
for service := range stat.services { }
w.services[server] = append(w.services[server], service) logger.Debug("Checking for dashboard availability")
if infos, err := checkDashboard(client, w.network); err != nil {
if err != ErrServiceUnknown {
stat.services["dashboard"] = map[string]string{"offline": err.Error()}
} }
} else {
stat.services["dashboard"] = infos.Report()
} }
// If a genesis block was found, load it into our configs // Feed and newly discovered information into the wizard
if protips.genesis != "" && w.conf.genesis == nil { w.lock.Lock()
genesis := new(core.Genesis) defer w.lock.Unlock()
if err := json.Unmarshal([]byte(protips.genesis), genesis); err != nil {
if genesis != "" && w.conf.genesis == nil {
g := new(core.Genesis)
if err := json.Unmarshal([]byte(genesis), g); err != nil {
log.Error("Failed to parse remote genesis", "err", err) log.Error("Failed to parse remote genesis", "err", err)
} else { } else {
w.conf.genesis = genesis w.conf.genesis = g
protips.network = genesis.Config.ChainId.Int64()
} }
} }
if protips.ethstats != "" { if ethstats != "" {
w.conf.ethstats = protips.ethstats w.conf.ethstats = ethstats
} }
w.conf.bootFull = protips.bootFull w.conf.bootFull = append(w.conf.bootFull, bootFull...)
w.conf.bootLight = protips.bootLight w.conf.bootLight = append(w.conf.bootLight, bootLight...)
// Print any collected stats and return return stat
stats.render()
} }
// serverStat is a collection of service configuration parameters and health // serverStat is a collection of service configuration parameters and health
...@@ -205,6 +237,9 @@ func (stats serverStats) render() { ...@@ -205,6 +237,9 @@ func (stats serverStats) render() {
} }
sort.Strings(services) sort.Strings(services)
if len(services) == 0 {
table.Append([]string{server, stats[server].address, "", "", ""})
}
for j, service := range services { for j, service := range services {
// Add an empty line between all services // Add an empty line between all services
if j > 0 { if j > 0 {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment