Improve logging and some little improvements

cmd/alltxt2http/main.go

@@ -20,15 +20,17 @@ import (
 //  "github.com/tzneal/ham-go/dxcc"
 //  "github.com/denzs/wsjtx-dashboards/shared/httpstuff"
   "github.com/hpcloud/tail"
+  "github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var alltxt      string
 var station     string
-var identifier    string
+var identifier  string
 var password    string
 var uri         string
 var minlen      int
 var readall     bool
+var trace       bool
 
 func usage() {
     fmt.Printf("Usage of %s:\n", os.Args[0])
@@ -43,6 +45,7 @@ func init() {
   flag.StringVar(&uri, "uri", "http://http-exporter", "uri to your http-exporter")
   flag.IntVar(&minlen, "minlen", 16, "minimal length for line to be pushed")
   flag.BoolVar(&readall, "readall", false, "submit whole file if nothing found at server")
+  flag.BoolVar(&trace, "trace", false, "log almost everything")
   flag.Parse()
 
   if alltxt == "" || station == "" || identifier == "" || uri == "" {
@@ -51,14 +54,28 @@ func init() {
     os.Exit(1)
   }
 
+  if trace {
+    log.SetLevel(log.TraceLevel)
+    log.Info("trace logging enabled")
+  } else {
+    log.Info("normal logging enabled")
+  }
+
   formatter := &log.TextFormatter{
     FullTimestamp: true,
   }
   log.SetFormatter(formatter)
+
+  log.Info("prometheus exporter enabled..")
+  go func () {
+    server := http.NewServeMux()
+    server.Handle("/metrics", promhttp.Handler())
+    http.ListenAndServe(":9123", server)
+  } ()
 }
 
 func getTimestampFromDb() (int64, bool) {
-  log.Trace("doing timestamp stuff..")
+  log.Trace("fetching last report from server")
 	var ts LastTs
 
   client := http.Client{
@@ -67,36 +84,32 @@ func getTimestampFromDb() (int64, bool) {
 
 	req, err := http.NewRequest(http.MethodGet, uri + "/timestamp/getLast?identifier=" + identifier, nil)
   req.SetBasicAuth(station, password)
-  log.Info("done req..")
 
 	if err != nil {
-		log.Fatal(err)
+    log.Errorf("error creating request: %s", err.Error())
 	}
 
 	req.Header.Set("X-Station", station)
 	req.Header.Set("X-Identifier", identifier)
 
-  log.Trace("done Headet.Set")
-
 	res, getErr := client.Do(req)
-  log.Trace("done client.Do")
 	if getErr != nil {
-		log.Fatal(getErr)
+    log.Errorf("error executing http request: %s", getErr.Error())
+    os.Exit(1)
 	}
 
   switch res.StatusCode {
     case 200:
-      log.Trace("200 OK, everything is fine :)")
+      log.Trace("200 OK")
     case 401:
-      log.Panic("Authentication failed! Please check entered station and password..")
+      log.Error("Authentication failed! Please check entered station and password..")
+      os.Exit(1)
     case 404:
-      log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier)
+      log.Tracef("404 for %s:%s", station, identifier)
       return 0, false
     case 502:
-      log.Info("Server reports internal problems (%d).. waiting 30 seconds to give it some time to breathe", res.StatusCode)
+      log.Errorf("Server reports internal problems (%d) please wait 30 seconds before trying again..", res.StatusCode)
-      time.Sleep(30 * time.Second)
+      os.Exit(1)
-      // FIXME what to do instead of panic? smart retry logic in main or even here?
-      log.Panic("bye! iam going home.. ")
     default:
       log.Panicf("uh well, there seems to be some serious shit going on: %d",res.StatusCode)
   }
@@ -120,15 +133,11 @@ func getTimestampFromDb() (int64, bool) {
 		log.Fatal(readErr)
 	}
 
-  log.Info("doing json stuff")
 	jsonErr := json.Unmarshal(body, &ts)
-  log.Info("done json stuff")
 	if jsonErr != nil {
 		log.Fatal(jsonErr)
 	}
 
-  log.Info("doing timestamp stuff done")
-
   return ts.Last, true
 }
 
@@ -173,58 +182,59 @@ func unix2wsjtx(ts int64) (string) {
 func postLine(batchmode bool, lines chan string) {
   // FIXME implement batchmode
 
-//  var l httpstuff.Lines
-
   for {
     select {
       case line := <- lines :
         l := Lines{}
         l.Text = append(l.Text, line)
-        log.Infof("line: %s", l.Text)
+        log.Tracef("line: %s", l.Text)
-
+
-    //var jsonStr = []byte(`{"title":"Buy cheese and bread for breakfast."}`)
+        jsonStr, err := json.Marshal(l)
-    jsonStr, err := json.Marshal(l)
+        if err != nil {
-    if err != nil {
+          log.Errorf("error marshaling json: ", err.Error())
-      log.Errorf("cant marshal json: ", err.Error())
+        }
-    }
+        req, err := http.NewRequest("POST", uri + "/lines/insert", bytes.NewBuffer(jsonStr))
-    req, err := http.NewRequest("POST", uri + "/lines/insert", bytes.NewBuffer(jsonStr))
+        if err != nil {
-    if err != nil {
+            log.Errorf("error creating request: %s", err.Error())
-        log.Errorf("crazy shit during newrequest", err.Error())
+        }
-    }
+        req.SetBasicAuth(station, password)
-    req.SetBasicAuth(station, password)
+	      req.Header.Set("X-Station", station)
-	  req.Header.Set("X-Station", station)
+	      req.Header.Set("X-Identifier", identifier)
-	  req.Header.Set("X-Identifier", identifier)
+        req.Header.Set("Content-Type", "application/json")
-    req.Header.Set("Content-Type", "application/json")
+
-
+        client := &http.Client{}
-    client := &http.Client{}
+        resp, err := client.Do(req)
-    resp, err := client.Do(req)
+        if err != nil {
-    if err != nil {
+            log.Errorf("error executing http request: %s", err.Error())
-        log.Errorf("crazy shit during client", err.Error())
+        }
-    }
+
-    defer resp.Body.Close()
+        // FIXME exit fatal if resposne != 200 OK
-
+        log.Tracef("response: %s", resp.Status)
-    log.Trace("response Status: ", resp.Status)
+//        body, _ := ioutil.ReadAll(resp.Body)
-    log.Trace("response Headers: ", resp.Header)
+//        log.Tracef("response: %s", string(body))
-    body, _ := ioutil.ReadAll(resp.Body)
+        resp.Body.Close()
-    log.Trace("response Body: ", string(body))
+        }
-    }
   }
   log.Trace("goodbye from postLine")
 }
 
 func readFile(offset int64, whence int, done chan bool) {
-  log.Trace("start readFile %s", alltxt)
+  workers := 128
 
-  lines := make(chan string ,2)
+  log.Tracef("start reading file")
 
-  go postLine((whence == 0), lines)
+  lines := make(chan string, workers)
-  go postLine((whence == 0), lines)
+
+  for i := 0; i < workers; i++ {
+    go postLine((whence == 0), lines)
+  }
 
   t, _:= tail.TailFile(alltxt, tail.Config{Follow: true, Location: &tail.SeekInfo{Offset: offset, Whence: whence},})
+  log.Info("scanning for lines..")
   for line := range t.Lines {
     // FIXME
-    // at least minlen
     // check for logline format!
+    // at least minlen
     lines <- line.Text
   }
 
@@ -240,8 +250,6 @@ func main(){
   var whence      int
   var foundoffset bool
 
-  log.Info("starting up..")
-
   offset = 0
 
   if !readall {
@@ -250,25 +258,25 @@ func main(){
     if foundts {
       log.Infof("last timestamp in db: %d", lastDbTime)
       wt := unix2wsjtx(lastDbTime)
-      log.Infof("searching %s for: %s",alltxt, wt)
+      log.Infof("searching %s for %s",alltxt, wt)
       offset, foundoffset = findPosition([]byte(wt))
       if foundoffset {
-        log.Trace("found %s at offset %d ", wt, offset)
+        log.Infof("found %s at offset %d", wt, offset)
         whence = 0
       } else {
         log.Warnf("timestamp %s NOT found in %s! have you deleted your ALL.TXT or used an old identifier?", wt, alltxt)
-      log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
+        log.Warnf("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
       }
     } else {
-      log.Warn("api has no data for %s:%s", station, identifier)
+      log.Warnf("api has no data for %s:%s", station, identifier)
-      log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
+      log.Warnf("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt)
     }
   } else {
     log.Info("readall mode enabled..")
     whence = 0
   }
 
-  log.Infof("offset: %d whence: %d", offset, whence)
+  log.Tracef("offset: %d whence: %d", offset, whence)
 
   done := make(chan bool)
   go readFile(offset, whence, done)

cmd/wsjtx-exporter/main.go

@@ -155,7 +155,7 @@ func lineRequestHandler() chan LineRequest {
 
   linerequests := make (chan LineRequest, runtime.NumCPU())
 
-  for i := 0; i < runtime.NumCPU(); i++ {
+  for i := 0; i < runtime.NumCPU()*16; i++ {
     log.Tracef("launching line worker %d", i)
     go lineParser(linerequests, i) 
   }

cmd/wsjtx-exporter/mysql.go

@@ -35,7 +35,7 @@ func getLast(station string, identifier string) (LastTs, error) {
 }
 
 func handleMysql(result wsjtx.Result) {
-  log.Info("starting mysq stuff..")
+  log.Info("starting mysql stuff..")
   db, dbDown := dbConn()
   if dbDown {
     log.Fatal("cant reach database..")

deploy.sh

@@ -14,3 +14,7 @@ env GOOS=windows GOARCH=amd64 go get ./...
 cp ~/go/bin/alltxt2http ~/Nextcloud/share/dl7le/binaries/linux64
 cp ~/go/bin/linux_arm/alltxt2http ~/Nextcloud/share/dl7le/binaries/raspi
 cp ~/go/bin/windows_amd64/alltxt2http.exe ~/Nextcloud/share/dl7le/binaries/win64
+
+ssh pi@192.168.2.122 systemctl stop --user alltxt2http.service
+scp ~/go/bin/linux_arm/alltxt2http pi@192.168.2.122:bin
+ssh pi@192.168.2.122 systemctl start --user alltxt2http.service

shared/wsjtx/wsjtx.go

@@ -158,7 +158,7 @@ func ScanLine(line string) (Result, bool) {
     result.Call = strings.Replace(tmp, ">", "", -1)
 
     result.Band = GetBand(element.Bandf)
-    result.Ent, found = dxcc.Lookup(result.Call)
+    result.Ent, found = dxcc.Lookup(result.Call)    // FIXME this is where the expensive stuff happens and should be cached..
     // FIXME result.Grid = qrz.lookup(result.Call) ;) or track in ALL.txt ^^
     if found && result.Band != "unknown" {
       result.Signal = element.Strength