From 5506d90c85358d8728c3ca1345206b3e6ef4a66f Mon Sep 17 00:00:00 2001 From: Sebastian Denz Date: Wed, 17 Mar 2021 21:57:05 +0100 Subject: [PATCH] Improve logging and some little improvements --- cmd/alltxt2http/main.go | 124 ++++++++++++++++++------------------ cmd/wsjtx-exporter/main.go | 2 +- cmd/wsjtx-exporter/mysql.go | 2 +- deploy.sh | 4 ++ shared/wsjtx/wsjtx.go | 2 +- 5 files changed, 69 insertions(+), 65 deletions(-) diff --git a/cmd/alltxt2http/main.go b/cmd/alltxt2http/main.go index bfc17dc..bb97fee 100644 --- a/cmd/alltxt2http/main.go +++ b/cmd/alltxt2http/main.go @@ -25,11 +25,12 @@ import ( var alltxt string var station string -var identifier string +var identifier string var password string var uri string var minlen int var readall bool +var trace bool func usage() { fmt.Printf("Usage of %s:\n", os.Args[0]) @@ -44,6 +45,7 @@ func init() { flag.StringVar(&uri, "uri", "http://http-exporter", "uri to your http-exporter") flag.IntVar(&minlen, "minlen", 16, "minimal length for line to be pushed") flag.BoolVar(&readall, "readall", false, "submit whole file if nothing found at server") + flag.BoolVar(&trace, "trace", false, "log almost everything") flag.Parse() if alltxt == "" || station == "" || identifier == "" || uri == "" { @@ -52,6 +54,13 @@ func init() { os.Exit(1) } + if trace { + log.SetLevel(log.TraceLevel) + log.Info("trace logging enabled") + } else { + log.Info("normal logging enabled") + } + formatter := &log.TextFormatter{ FullTimestamp: true, } @@ -66,7 +75,7 @@ func init() { } func getTimestampFromDb() (int64, bool) { - log.Trace("doing timestamp stuff..") + log.Trace("fetching last report from server") var ts LastTs client := http.Client{ @@ -75,36 +84,32 @@ func getTimestampFromDb() (int64, bool) { req, err := http.NewRequest(http.MethodGet, uri + "/timestamp/getLast?identifier=" + identifier, nil) req.SetBasicAuth(station, password) - log.Info("done req..") if err != nil { - log.Fatal(err) + log.Errorf("error creating request: %s", err.Error()) } req.Header.Set("X-Station", station) req.Header.Set("X-Identifier", identifier) - log.Trace("done Headet.Set") - res, getErr := client.Do(req) - log.Trace("done client.Do") if getErr != nil { - log.Fatal(getErr) + log.Errorf("error executing http request: %s", getErr.Error()) + os.Exit(1) } switch res.StatusCode { case 200: - log.Trace("200 OK, everything is fine :)") + log.Trace("200 OK") case 401: - log.Panic("Authentication failed! Please check entered station and password..") + log.Error("Authentication failed! Please check entered station and password..") + os.Exit(1) case 404: - log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier) + log.Tracef("404 for %s:%s", station, identifier) return 0, false case 502: - log.Info("Server reports internal problems (%d).. waiting 30 seconds to give it some time to breathe", res.StatusCode) - time.Sleep(30 * time.Second) - // FIXME what to do instead of panic? smart retry logic in main or even here? - log.Panic("bye! iam going home.. ") + log.Errorf("Server reports internal problems (%d) please wait 30 seconds before trying again..", res.StatusCode) + os.Exit(1) default: log.Panicf("uh well, there seems to be some serious shit going on: %d",res.StatusCode) } @@ -128,15 +133,11 @@ func getTimestampFromDb() (int64, bool) { log.Fatal(readErr) } - log.Info("doing json stuff") jsonErr := json.Unmarshal(body, &ts) - log.Info("done json stuff") if jsonErr != nil { log.Fatal(jsonErr) } - log.Info("doing timestamp stuff done") - return ts.Last, true } @@ -181,58 +182,59 @@ func unix2wsjtx(ts int64) (string) { func postLine(batchmode bool, lines chan string) { // FIXME implement batchmode -// var l httpstuff.Lines - for { select { case line := <- lines : l := Lines{} l.Text = append(l.Text, line) - log.Infof("line: %s", l.Text) - - //var jsonStr = []byte(`{"title":"Buy cheese and bread for breakfast."}`) - jsonStr, err := json.Marshal(l) - if err != nil { - log.Errorf("cant marshal json: ", err.Error()) - } - req, err := http.NewRequest("POST", uri + "/lines/insert", bytes.NewBuffer(jsonStr)) - if err != nil { - log.Errorf("crazy shit during newrequest", err.Error()) - } - req.SetBasicAuth(station, password) - req.Header.Set("X-Station", station) - req.Header.Set("X-Identifier", identifier) - req.Header.Set("Content-Type", "application/json") - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - log.Errorf("crazy shit during client", err.Error()) - } - defer resp.Body.Close() - - log.Trace("response Status: ", resp.Status) - log.Trace("response Headers: ", resp.Header) - body, _ := ioutil.ReadAll(resp.Body) - log.Trace("response Body: ", string(body)) - } + log.Tracef("line: %s", l.Text) + + jsonStr, err := json.Marshal(l) + if err != nil { + log.Errorf("error marshaling json: ", err.Error()) + } + req, err := http.NewRequest("POST", uri + "/lines/insert", bytes.NewBuffer(jsonStr)) + if err != nil { + log.Errorf("error creating request: %s", err.Error()) + } + req.SetBasicAuth(station, password) + req.Header.Set("X-Station", station) + req.Header.Set("X-Identifier", identifier) + req.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + log.Errorf("error executing http request: %s", err.Error()) + } + + // FIXME exit fatal if resposne != 200 OK + log.Tracef("response: %s", resp.Status) +// body, _ := ioutil.ReadAll(resp.Body) +// log.Tracef("response: %s", string(body)) + resp.Body.Close() + } } log.Trace("goodbye from postLine") } func readFile(offset int64, whence int, done chan bool) { - log.Trace("start readFile %s", alltxt) + workers := 128 - lines := make(chan string ,2) + log.Tracef("start reading file") - go postLine((whence == 0), lines) - go postLine((whence == 0), lines) + lines := make(chan string, workers) + + for i := 0; i < workers; i++ { + go postLine((whence == 0), lines) + } t, _:= tail.TailFile(alltxt, tail.Config{Follow: true, Location: &tail.SeekInfo{Offset: offset, Whence: whence},}) + log.Info("scanning for lines..") for line := range t.Lines { // FIXME - // at least minlen // check for logline format! + // at least minlen lines <- line.Text } @@ -248,8 +250,6 @@ func main(){ var whence int var foundoffset bool - log.Info("starting up..") - offset = 0 if !readall { @@ -258,25 +258,25 @@ func main(){ if foundts { log.Infof("last timestamp in db: %d", lastDbTime) wt := unix2wsjtx(lastDbTime) - log.Infof("searching %s for: %s",alltxt, wt) + log.Infof("searching %s for %s",alltxt, wt) offset, foundoffset = findPosition([]byte(wt)) if foundoffset { - log.Trace("found %s at offset %d ", wt, offset) + log.Infof("found %s at offset %d", wt, offset) whence = 0 } else { log.Warnf("timestamp %s NOT found in %s! have you deleted your ALL.TXT or used an old identifier?", wt, alltxt) - log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt) + log.Warnf("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt) } } else { - log.Warn("api has no data for %s:%s", station, identifier) - log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt) + log.Warnf("api has no data for %s:%s", station, identifier) + log.Warnf("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt) } } else { log.Info("readall mode enabled..") whence = 0 } - log.Infof("offset: %d whence: %d", offset, whence) + log.Tracef("offset: %d whence: %d", offset, whence) done := make(chan bool) go readFile(offset, whence, done) diff --git a/cmd/wsjtx-exporter/main.go b/cmd/wsjtx-exporter/main.go index 95c616c..2e6f74f 100644 --- a/cmd/wsjtx-exporter/main.go +++ b/cmd/wsjtx-exporter/main.go @@ -155,7 +155,7 @@ func lineRequestHandler() chan LineRequest { linerequests := make (chan LineRequest, runtime.NumCPU()) - for i := 0; i < runtime.NumCPU(); i++ { + for i := 0; i < runtime.NumCPU()*16; i++ { log.Tracef("launching line worker %d", i) go lineParser(linerequests, i) } diff --git a/cmd/wsjtx-exporter/mysql.go b/cmd/wsjtx-exporter/mysql.go index b48b1f3..ec4b154 100644 --- a/cmd/wsjtx-exporter/mysql.go +++ b/cmd/wsjtx-exporter/mysql.go @@ -35,7 +35,7 @@ func getLast(station string, identifier string) (LastTs, error) { } func handleMysql(result wsjtx.Result) { - log.Info("starting mysq stuff..") + log.Info("starting mysql stuff..") db, dbDown := dbConn() if dbDown { log.Fatal("cant reach database..") diff --git a/deploy.sh b/deploy.sh index 52f36df..c270596 100755 --- a/deploy.sh +++ b/deploy.sh @@ -14,3 +14,7 @@ env GOOS=windows GOARCH=amd64 go get ./... cp ~/go/bin/alltxt2http ~/Nextcloud/share/dl7le/binaries/linux64 cp ~/go/bin/linux_arm/alltxt2http ~/Nextcloud/share/dl7le/binaries/raspi cp ~/go/bin/windows_amd64/alltxt2http.exe ~/Nextcloud/share/dl7le/binaries/win64 + +ssh pi@192.168.2.122 systemctl stop --user alltxt2http.service +scp ~/go/bin/linux_arm/alltxt2http pi@192.168.2.122:bin +ssh pi@192.168.2.122 systemctl start --user alltxt2http.service diff --git a/shared/wsjtx/wsjtx.go b/shared/wsjtx/wsjtx.go index 21c8ee5..c1e150f 100644 --- a/shared/wsjtx/wsjtx.go +++ b/shared/wsjtx/wsjtx.go @@ -158,7 +158,7 @@ func ScanLine(line string) (Result, bool) { result.Call = strings.Replace(tmp, ">", "", -1) result.Band = GetBand(element.Bandf) - result.Ent, found = dxcc.Lookup(result.Call) + result.Ent, found = dxcc.Lookup(result.Call) // FIXME this is where the expensive stuff happens and should be cached.. // FIXME result.Grid = qrz.lookup(result.Call) ;) or track in ALL.txt ^^ if found && result.Band != "unknown" { result.Signal = element.Strength