working exporter and httpclient

Dockerfile

@@ -0,0 +1,8 @@
+from golang:1.15.0
+
+RUN mkdir /wsjtx-exporter
+ADD . /wsjtx-exporter
+WORKDIR /wsjtx-exporter/cmd/wsjtx-exporter
+RUN go build 
+
+CMD ["/wsjtx-exporter/cmd/wsjtx-exporter/wsjtx-exporter"]

cmd/alltxt2http/main.go

@@ -58,7 +58,7 @@ func init() {
 }
 
 func getTimestampFromDb() (int64, bool) {
-  log.Info("doing timestamp stuff..")
+  log.Trace("doing timestamp stuff..")
 	var ts LastTs
 
   client := http.Client{
@@ -66,32 +66,50 @@ func getTimestampFromDb() (int64, bool) {
 	}
 
 	req, err := http.NewRequest(http.MethodGet, uri + "/timestamp/getLast?identifier=" + identifier, nil)
+  req.SetBasicAuth(station, password)
   log.Info("done req..")
 
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	req.Header.Set("X-STATION", station)
+	req.Header.Set("X-Station", station)
-	req.Header.Set("X-IDENTIFIER", identifier)
+	req.Header.Set("X-Identifier", identifier)
 
-  log.Info("done Headet.Set")
+  log.Trace("done Headet.Set")
 
 	res, getErr := client.Do(req)
-  log.Info("done client.Do")
+  log.Trace("done client.Do")
 	if getErr != nil {
 		log.Fatal(getErr)
 	}
 
-  if res.StatusCode != http.StatusOK {
+  switch res.StatusCode {
-      if res.StatusCode == 404 {
+    case 200:
-        // FIXME on windows show message to upload all.txt
+      log.Trace("200 OK, everything is fine :)")
+    case 401:
+      log.Panic("Authentication failed! Please check entered station and password..")
+    case 404:
       log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier)
       return 0, false
+    case 502:
+      log.Info("Server reports internal problems (%d).. waiting 30 seconds to give it some time to breathe", res.StatusCode)
+      time.Sleep(30 * time.Second)
+      // FIXME what to do instead of panic? smart retry logic in main or even here?
+      log.Panic("bye! iam going home.. ")
+    default:
+      log.Panicf("uh well, there seems to be some serious shit going on: %d",res.StatusCode)
   }
-      fmt.Println("Non-OK HTTP status:", res.StatusCode)
+
-      log.Panic("uh well, there seems to be some serious shit going on...")
+//  if res.StatusCode != http.StatusOK {
-  }
+//      if res.StatusCode == 404 {
+//        // FIXME on windows show message to upload all.txt
+//        log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier)
+//        return 0, false
+//      }
+//      fmt.Println("Non-OK HTTP status:", res.StatusCode)
+//      log.Panic("uh well, there seems to be some serious shit going on...")
+//  }
 
 	if res.Body != nil {
 		defer res.Body.Close()
@@ -173,8 +191,9 @@ func postLine(batchmode bool, lines chan string) {
     if err != nil {
         log.Errorf("crazy shit during newrequest", err.Error())
     }
-	  req.Header.Set("X-STATION", station)
+    req.SetBasicAuth(station, password)
-	  req.Header.Set("X-IDENTIFIER", identifier)
+	  req.Header.Set("X-Station", station)
+	  req.Header.Set("X-Identifier", identifier)
     req.Header.Set("Content-Type", "application/json")
 
     client := &http.Client{}

cmd/http-exporter/main.go → cmd/wsjtx-exporter/main.go

@@ -45,7 +45,7 @@ func usage() {
 func init() {
   flag.StringVar(&mysqlHost, "host", "db", "name/ip of mysql host")
   flag.StringVar(&mysqlDb, "db", "digimode_stats", "db name")
-  flag.StringVar(&mysqlUser, "user", "wsjtx", "mysql username")
+  flag.StringVar(&mysqlUser, "user", "stationmonitor", "mysql username")
   flag.StringVar(&mysqlPass, "pass", "secret", "mysql password")
   flag.StringVar(&mysqlTable, "table", "wsjtx_all_txt", "mysql table name")
   flag.StringVar(&metricPath, "metricPath", "/metrics", "path for prometheus metric endpoint")
@@ -101,13 +101,16 @@ func init() {
 
 
 func handleGetLastRequest(w http.ResponseWriter, r *http.Request) {
-  xstation  := r.Header.Get("X-STATION")
+  xstation  := r.Header.Get("X-Station")
-//  xproxystation := r.Header.Get("X-PROXY-STATION")
+  xproxystation := r.Header.Get("X-Proxy-Station")
-  xidentifier := r.Header.Get("X-IDENTIFIER")
+  xidentifier := r.Header.Get("X-Identifier")
-  fmt.Printf("X-STATION: %s\n", xstation)
+  fmt.Printf("X-Station: %s\n", xstation)
-  fmt.Printf("X-IDENTIFIER: %s\n", xidentifier)
+  fmt.Printf("X-Identifier: %s\n", xidentifier)
 
   // FIXME check for existence! and compare X-STATION against X-PROXY-STATION!
+  if xproxystation != xstation {
+    log.Errorf("ehmm... user and webserver have different opinion on station X-Station: , X-Proxy-Station:", xstation, xproxystation)
+  }
  
   last, err := getLast(xstation, xidentifier)
   if err != nil {
@@ -163,11 +166,14 @@ func lineRequestHandler() chan LineRequest {
 func (lrm *LineRequestManager) handleLinesRequest(w http.ResponseWriter, r *http.Request) {
   var l Lines
 
-  xstation  := r.Header.Get("X-STATION")
+  xstation  := r.Header.Get("X-Station")
-//  xproxystation := r.Header.Get("X-PROXY-STATION")
+  xproxystation := r.Header.Get("X-Proxy-Station")
-  xidentifier := r.Header.Get("X-IDENTIFIER")
+  xidentifier := r.Header.Get("X-Identifier")
 
   // FIXME check for existence! and compare X-STATION against X-PROXY-STATION!
+  if xproxystation != xstation {
+    log.Errorf("ehmm... user and webserver have different opinion on station X-Station: , X-Proxy-Station:", xstation, xproxystation)
+  }
   
   b, err := ioutil.ReadAll(r.Body)
   defer r.Body.Close()

cmd/http-exporter/mysql.go → cmd/wsjtx-exporter/mysql.go

@@ -18,9 +18,7 @@ func getLast(station string, identifier string) (LastTs, error) {
   }
   defer db.Close()
 
-  log.Info("starting query...")
   err := db.QueryRow("SELECT UNIX_TIMESTAMP(ts) FROM wsjtx_all_txt WHERE station = ? AND identifier = ? ORDER BY ts DESC LIMIT 1", station, identifier).Scan(&result.Last)
-  log.Info("finished query...")
   if err != nil {
     // FIXME 
     if err.Error() == "sql: no rows in result set" {
@@ -51,15 +49,12 @@ func handleMysql(result wsjtx.Result) {
 
   if err != nil {
     log.WithFields(log.Fields{"warning":err.Error()}).Warn("error while executing prepared statement..")
-  } else {
-    log.Info("insert done! :)")
   }
 }
 
 
 func init_db() {
   var cnt int
-  log.Debug("init_db()")
   db, _ := dbConn()
 
   row := db.QueryRow("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_NAME LIKE '%" + mysqlTable + "%';")
@@ -74,21 +69,44 @@ func init_db() {
   }
 
   if cnt < 1 {
-    qry := "CREATE TABLE IF NOT EXISTS " + mysqlTable + " (" +
+    qry := "CREATE TABLE wsjtx_all_txt (" +
-                        "ts timestamp NOT NULL," +
+              "ts datetime NOT NULL," +
-                        "station VARCHAR(16) NOT NULL," +
+              "station varchar(16) NOT NULL," +
-                        "callsign VARCHAR(16) NOT NULL," +
+              "callsign varchar(16) NOT NULL," +
-                        "band VARCHAR(10) NOT NULL," +
+              "band varchar(10) NOT NULL," +
-                        "continent VARCHAR(32) NOT NULL," +
+              "continent varchar(32) NOT NULL," +
-                        "mode VARCHAR(16) NOT NULL," +
+              "mode varchar(16) NOT NULL," +
-                        "dxcc VARCHAR(128) NOT NULL," +
+              "dxcc varchar(128) NOT NULL," +
-                        "geohash VARCHAR(16) NOT NULL," +
+              "geohash varchar(16) NOT NULL," +
-                        "report TINYINT NOT NULL," +
+              "report tinyint(4) NOT NULL," +
-                        "cqzone INT NOT NULL," +
+              "cqzone int(11) NOT NULL," +
-                        "ituzone INT NOT NULL," +
+              "ituzone int(11) NOT NULL," +
-                        "rx TINYINT NOT NULL," +
+              "rx tinyint(4) NOT NULL," +
-                        "PRIMARY KEY UC_" + mysqlTable + "(ts, station, callsign)," +
+              "identifier varchar(32) DEFAULT NULL," +
-                        "INDEX idx_dxcc (dxcc));"
+              "PRIMARY KEY (ts,station,callsign)," +
+              "KEY idx_dxcc (dxcc)," +
+              "KEY idx_continent (continent)," +
+              "KEY idx_station (station)," +
+              "KEY idx_band (band)," +
+              "KEY idx_station_identifier (station,identifier)," +
+              "KEY idx_ts_station_band (ts,station,band)," +
+              "KEY idx_ts (ts));";
+
+//    qry := "CREATE TABLE IF NOT EXISTS " + mysqlTable + " (" +
+//                        "ts timestamp NOT NULL," +
+//                        "station VARCHAR(16) NOT NULL," +
+//                        "callsign VARCHAR(16) NOT NULL," +
+//                        "band VARCHAR(10) NOT NULL," +
+//                        "continent VARCHAR(32) NOT NULL," +
+//                        "mode VARCHAR(16) NOT NULL," +
+//                        "dxcc VARCHAR(128) NOT NULL," +
+//                        "geohash VARCHAR(16) NOT NULL," +
+//                        "report TINYINT NOT NULL," +
+//                        "cqzone INT NOT NULL," +
+//                        "ituzone INT NOT NULL," +
+//                        "rx TINYINT NOT NULL," +
+//                        "PRIMARY KEY UC_" + mysqlTable + "(ts, station, callsign)," +
+//                        "INDEX idx_dxcc (dxcc));"
     log.WithFields(log.Fields{"query":qry}).Debug("creating database..")
     _, err := db.Exec(qry)
   	if err != nil {
@@ -103,7 +121,9 @@ func init_db() {
 
 func dbConn() (db *sql.DB, err bool){
   //db, er := sql.Open("mysql", mysql_user + ":" + mysql_pass + "@tcp(" + mysql_host + ")/" + mysql_db + "?parseTime=true&time_zone=%27UTC%27")
-  db, er := sql.Open("mysql", mysqlUser + ":" + mysqlPass + "@tcp(" + mysqlHost + ")/" + mysqlDb)
+  dbUri :=  mysqlUser + ":" + mysqlPass + "@tcp(" + mysqlHost + ")/" + mysqlDb
+  log.Tracef("dbUri: %s", dbUri)
+  db, er := sql.Open("mysql", dbUri)
   if er != nil {
     log.Error("db not reachable: %s",err)
     return nil, true
@@ -111,7 +131,7 @@ func dbConn() (db *sql.DB, err bool){
   
   pingerr := db.Ping()
   if pingerr != nil {
-    log.Error("db not pingable..")
+    log.Error("db not pingable: ",pingerr.Error())
     return nil, true
   }
 

deploy.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+app="wsjtx-exporter"
+
+docker build -t 10.0.73.1:5000/$app:latest . || exit 1
+#export TAG=$(docker images|grep -v REPO|head -n1|awk '{print $3}')
+#docker tag $TAG 10.0.73.1:5000/$app:latest || exit 1
+docker push 10.0.73.1:5000/$app:latest || exit 1
+
+go get ./...
+env GOOS=linux GOARCH=arm GOARM=5 go get ./...
+env GOOS=windows GOARCH=amd64 go get ./...
+
+cp ~/go/bin/alltxt2http ~/Nextcloud/share/dl7le/binaries/linux64
+cp ~/go/bin/linux_arm/alltxt2http ~/Nextcloud/share/dl7le/binaries/raspi
+cp ~/go/bin/windows_amd64/alltxt2http.exe ~/Nextcloud/share/dl7le/binaries/win64

wsjtx_all_txt.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS wsjtx_all_txt (
+  ts timestamp NOT NULL,
+  station VARCHAR(16) NOT NULL,
+  callsign VARCHAR(16) NOT NULL,
+  band VARCHAR(10) NOT NULL,
+  continent VARCHAR(32) NOT NULL,
+  mode VARCHAR(16) NOT NULL,
+  dxcc VARCHAR(128) NOT NULL,
+  geohash VARCHAR(16) NOT NULL,
+  report TINYINT NOT NULL,
+  cqzone INT NOT NULL,
+  ituzone INT NOT NULL,
+  rx TINYINT NOT NULL,
+  PRIMARY KEY PK_wsjtx_all_txt (ts, station, callsign),
+  INDEX idx_dxcc (dxcc)
+);