ixyd
/
wsjtx-exporter
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity
Browse Source

Improve logging and some little improvements

master
Sebastian Denz 4 years ago
parent
a27ee0b8cc
commit
5506d90c85
5 changed files with 69 additions and 65 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 124
      cmd/alltxt2http/main.go
  2. 2
      cmd/wsjtx-exporter/main.go
  3. 2
      cmd/wsjtx-exporter/mysql.go
  4. 4
      deploy.sh
  5. 2
      shared/wsjtx/wsjtx.go

124
cmd/alltxt2http/main.go
View File

@ -25,11 +25,12 @@ import (
var alltxt string
var station string
var identifier string
var identifier string
var password string
var uri string
var minlen int
var readall bool
var trace bool
func usage() {
fmt.Printf("Usage of %s:\n", os.Args[0])
@ -44,6 +45,7 @@ func init() {
flag.StringVar(&uri, "uri", "http://http-exporter", "uri to your http-exporter")
flag.IntVar(&minlen, "minlen", 16, "minimal length for line to be pushed")
flag.BoolVar(&readall, "readall", false, "submit whole file if nothing found at server")
flag.BoolVar(&trace, "trace", false, "log almost everything")
flag.Parse()
if alltxt == "" || station == "" || identifier == "" || uri == "" {
@ -52,6 +54,13 @@ func init() {
os.Exit(1)
}
if trace {
log.SetLevel(log.TraceLevel)
log.Info("trace logging enabled")
} else {
log.Info("normal logging enabled")
}
formatter := &log.TextFormatter{
FullTimestamp: true,
}
@ -66,7 +75,7 @@ func init() {
}
func getTimestampFromDb() (int64, bool) {
log.Trace("doing timestamp stuff..")
log.Trace("fetching last report from server")
var ts LastTs
client := http.Client{
@ -75,36 +84,32 @@ func getTimestampFromDb() (int64, bool) {
req, err := http.NewRequest(http.MethodGet, uri + "/timestamp/getLast?identifier=" + identifier, nil)
req.SetBasicAuth(station, password)
log.Info("done req..")
if err != nil {
log.Fatal(err)
log.Errorf("error creating request: %s", err.Error())
}
req.Header.Set("X-Station", station)
req.Header.Set("X-Identifier", identifier)
log.Trace("done Headet.Set")
res, getErr := client.Do(req)
log.Trace("done client.Do")
if getErr != nil {
log.Fatal(getErr)
log.Errorf("error executing http request: %s", getErr.Error())
os.Exit(1)
}
switch res.StatusCode {
case 200:
log.Trace("200 OK, everything is fine :)")
log.Trace("200 OK")
case 401:
log.Panic("Authentication failed! Please check entered station and password..")
log.Error("Authentication failed! Please check entered station and password..")
os.Exit(1)
case 404:
log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier)
log.Tracef("404 for %s:%s", station, identifier)
return 0, false
case 502:
log.Info("Server reports internal problems (%d).. waiting 30 seconds to give it some time to breathe", res.StatusCode)
time.Sleep(30 * time.Second)
// FIXME what to do instead of panic? smart retry logic in main or even here?
log.Panic("bye! iam going home.. ")
log.Errorf("Server reports internal problems (%d) please wait 30 seconds before trying again..", res.StatusCode)
os.Exit(1)
default:
log.Panicf("uh well, there seems to be some serious shit going on: %d",res.StatusCode)
}
@ -128,15 +133,11 @@ func getTimestampFromDb() (int64, bool) {
log.Fatal(readErr)
}
log.Info("doing json stuff")
jsonErr := json.Unmarshal(body, &ts)
log.Info("done json stuff")
if jsonErr != nil {
log.Fatal(jsonErr)
}
log.Info("doing timestamp stuff done")
return ts.Last, true
}
@ -181,58 +182,59 @@ func unix2wsjtx(ts int64) (string) {
func postLine(batchmode bool, lines chan string) {
// FIXME implement batchmode
// var l httpstuff.Lines
for {
select {
case line := <- lines :
l := Lines{}
l.Text = append(l.Text, line)
log.Infof("line: %s", l.Text)
//var jsonStr = []byte(`{"title":"Buy cheese and bread for breakfast."}`)
jsonStr, err := json.Marshal(l)
if err != nil {
log.Errorf("cant marshal json: ", err.Error())
}
req, err := http.NewRequest("POST", uri + "/lines/insert", bytes.NewBuffer(jsonStr))
if err != nil {
log.Errorf("crazy shit during newrequest", err.Error())
}
req.SetBasicAuth(station, password)
req.Header.Set("X-Station", station)
req.Header.Set("X-Identifier", identifier)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Errorf("crazy shit during client", err.Error())
}
defer resp.Body.Close()
log.Trace("response Status: ", resp.Status)
log.Trace("response Headers: ", resp.Header)
body, _ := ioutil.ReadAll(resp.Body)
log.Trace("response Body: ", string(body))
}
log.Tracef("line: %s", l.Text)
jsonStr, err := json.Marshal(l)
if err != nil {
log.Errorf("error marshaling json: ", err.Error())
}
req, err := http.NewRequest("POST", uri + "/lines/insert", bytes.NewBuffer(jsonStr))
if err != nil {
log.Errorf("error creating request: %s", err.Error())
}
req.SetBasicAuth(station, password)
req.Header.Set("X-Station", station)
req.Header.Set("X-Identifier", identifier)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Errorf("error executing http request: %s", err.Error())
}
// FIXME exit fatal if resposne != 200 OK
log.Tracef("response: %s", resp.Status)
// body, _ := ioutil.ReadAll(resp.Body)
// log.Tracef("response: %s", string(body))
resp.Body.Close()
}
}
log.Trace("goodbye from postLine")
}
func readFile(offset int64, whence int, done chan bool) {
log.Trace("start readFile %s", alltxt)
workers := 128
lines := make(chan string ,2)
log.Tracef("start reading file")
go postLine((whence == 0), lines)
go postLine((whence == 0), lines)
lines := make(chan string, workers)
for i := 0; i < workers; i++ {
go postLine((whence == 0), lines)
}
t, _:= tail.TailFile(alltxt, tail.Config{Follow: true, Location: &tail.SeekInfo{Offset: offset, Whence: whence},})
log.Info("scanning for lines..")
for line := range t.Lines {
// FIXME
// at least minlen
// check for logline format!
// at least minlen
lines <- line.Text
}
@ -248,8 +250,6 @@ func main(){
var whence int
var foundoffset bool
log.Info("starting up..")
offset = 0
if !readall {
@ -258,25 +258,25 @@ func main(){
if foundts {
log.Infof("last timestamp in db: %d", lastDbTime)
wt := unix2wsjtx(lastDbTime)
log.Infof("searching %s for: %s",alltxt, wt)
log.Infof("searching %s for %s",alltxt, wt)
offset, foundoffset = findPosition([]byte(wt))
if foundoffset {
log.Trace("found %s at offset %d ", wt, offset)
log.Infof("found %s at offset %d", wt, offset)
whence = 0
} else {
log.Warnf("timestamp %s NOT found in %s! have you deleted your ALL.TXT or used an old identifier?", wt, alltxt)
log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
log.Warnf("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
}
} else {
log.Warn("api has no data for %s:%s", station, identifier)
log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
log.Warnf("api has no data for %s:%s", station, identifier)
log.Warnf("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
}
} else {
log.Info("readall mode enabled..")
whence = 0
}
log.Infof("offset: %d whence: %d", offset, whence)
log.Tracef("offset: %d whence: %d", offset, whence)
done := make(chan bool)
go readFile(offset, whence, done)

2
cmd/wsjtx-exporter/main.go
View File

@ -155,7 +155,7 @@ func lineRequestHandler() chan LineRequest {
linerequests := make (chan LineRequest, runtime.NumCPU())
for i := 0; i < runtime.NumCPU(); i++ {
for i := 0; i < runtime.NumCPU()*16; i++ {
log.Tracef("launching line worker %d", i)
go lineParser(linerequests, i)
}

2
cmd/wsjtx-exporter/mysql.go
View File

@ -35,7 +35,7 @@ func getLast(station string, identifier string) (LastTs, error) {
}
func handleMysql(result wsjtx.Result) {
log.Info("starting mysq stuff..")
log.Info("starting mysql stuff..")
db, dbDown := dbConn()
if dbDown {
log.Fatal("cant reach database..")

4
deploy.sh
View File

@ -14,3 +14,7 @@ env GOOS=windows GOARCH=amd64 go get ./...
cp ~/go/bin/alltxt2http ~/Nextcloud/share/dl7le/binaries/linux64
cp ~/go/bin/linux_arm/alltxt2http ~/Nextcloud/share/dl7le/binaries/raspi
cp ~/go/bin/windows_amd64/alltxt2http.exe ~/Nextcloud/share/dl7le/binaries/win64
ssh pi@192.168.2.122 systemctl stop --user alltxt2http.service
scp ~/go/bin/linux_arm/alltxt2http pi@192.168.2.122:bin
ssh pi@192.168.2.122 systemctl start --user alltxt2http.service

2
shared/wsjtx/wsjtx.go
View File

@ -158,7 +158,7 @@ func ScanLine(line string) (Result, bool) {
result.Call = strings.Replace(tmp, ">", "", -1)
result.Band = GetBand(element.Bandf)
result.Ent, found = dxcc.Lookup(result.Call)
result.Ent, found = dxcc.Lookup(result.Call) // FIXME this is where the expensive stuff happens and should be cached..
// FIXME result.Grid = qrz.lookup(result.Call) ;) or track in ALL.txt ^^
if found && result.Band != "unknown" {
result.Signal = element.Strength

Write Preview
Loading…
Cancel
Save