From 69850cf0d5838c42a9167d40a243fbdde11d048b Mon Sep 17 00:00:00 2001 From: Sebastian Denz Date: Wed, 9 Dec 2020 00:38:05 +0100 Subject: [PATCH] working exporter and httpclient --- Dockerfile | 8 +++ readme.md => README.md | 0 cmd/alltxt2http/main.go | 49 +++++++++----- cmd/{http-exporter => wsjtx-exporter}/main.go | 24 ++++--- .../mysql.go | 64 ++++++++++++------- .../prometheus.go | 0 .../types.go | 0 deploy.sh | 16 +++++ wsjtx_all_txt.sql | 16 +++++ 9 files changed, 131 insertions(+), 46 deletions(-) create mode 100644 Dockerfile rename readme.md => README.md (100%) rename cmd/{http-exporter => wsjtx-exporter}/main.go (86%) rename cmd/{http-exporter => wsjtx-exporter}/mysql.go (57%) rename cmd/{http-exporter => wsjtx-exporter}/prometheus.go (100%) rename cmd/{http-exporter => wsjtx-exporter}/types.go (100%) create mode 100755 deploy.sh create mode 100644 wsjtx_all_txt.sql diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8d7fb70 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +from golang:1.15.0 + +RUN mkdir /wsjtx-exporter +ADD . /wsjtx-exporter +WORKDIR /wsjtx-exporter/cmd/wsjtx-exporter +RUN go build + +CMD ["/wsjtx-exporter/cmd/wsjtx-exporter/wsjtx-exporter"] diff --git a/readme.md b/README.md similarity index 100% rename from readme.md rename to README.md diff --git a/cmd/alltxt2http/main.go b/cmd/alltxt2http/main.go index c26ed83..9da1ae7 100644 --- a/cmd/alltxt2http/main.go +++ b/cmd/alltxt2http/main.go @@ -58,7 +58,7 @@ func init() { } func getTimestampFromDb() (int64, bool) { - log.Info("doing timestamp stuff..") + log.Trace("doing timestamp stuff..") var ts LastTs client := http.Client{ @@ -66,33 +66,51 @@ func getTimestampFromDb() (int64, bool) { } req, err := http.NewRequest(http.MethodGet, uri + "/timestamp/getLast?identifier=" + identifier, nil) + req.SetBasicAuth(station, password) log.Info("done req..") if err != nil { log.Fatal(err) } - req.Header.Set("X-STATION", station) - req.Header.Set("X-IDENTIFIER", identifier) + req.Header.Set("X-Station", station) + req.Header.Set("X-Identifier", identifier) - log.Info("done Headet.Set") + log.Trace("done Headet.Set") res, getErr := client.Do(req) - log.Info("done client.Do") + log.Trace("done client.Do") if getErr != nil { log.Fatal(getErr) } - if res.StatusCode != http.StatusOK { - if res.StatusCode == 404 { - // FIXME on windows show message to upload all.txt - log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier) - return 0, false - } - fmt.Println("Non-OK HTTP status:", res.StatusCode) - log.Panic("uh well, there seems to be some serious shit going on...") + switch res.StatusCode { + case 200: + log.Trace("200 OK, everything is fine :)") + case 401: + log.Panic("Authentication failed! Please check entered station and password..") + case 404: + log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier) + return 0, false + case 502: + log.Info("Server reports internal problems (%d).. waiting 30 seconds to give it some time to breathe", res.StatusCode) + time.Sleep(30 * time.Second) + // FIXME what to do instead of panic? smart retry logic in main or even here? + log.Panic("bye! iam going home.. ") + default: + log.Panicf("uh well, there seems to be some serious shit going on: %d",res.StatusCode) } +// if res.StatusCode != http.StatusOK { +// if res.StatusCode == 404 { +// // FIXME on windows show message to upload all.txt +// log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier) +// return 0, false +// } +// fmt.Println("Non-OK HTTP status:", res.StatusCode) +// log.Panic("uh well, there seems to be some serious shit going on...") +// } + if res.Body != nil { defer res.Body.Close() } @@ -173,8 +191,9 @@ func postLine(batchmode bool, lines chan string) { if err != nil { log.Errorf("crazy shit during newrequest", err.Error()) } - req.Header.Set("X-STATION", station) - req.Header.Set("X-IDENTIFIER", identifier) + req.SetBasicAuth(station, password) + req.Header.Set("X-Station", station) + req.Header.Set("X-Identifier", identifier) req.Header.Set("Content-Type", "application/json") client := &http.Client{} diff --git a/cmd/http-exporter/main.go b/cmd/wsjtx-exporter/main.go similarity index 86% rename from cmd/http-exporter/main.go rename to cmd/wsjtx-exporter/main.go index 62bedf5..95c616c 100644 --- a/cmd/http-exporter/main.go +++ b/cmd/wsjtx-exporter/main.go @@ -45,7 +45,7 @@ func usage() { func init() { flag.StringVar(&mysqlHost, "host", "db", "name/ip of mysql host") flag.StringVar(&mysqlDb, "db", "digimode_stats", "db name") - flag.StringVar(&mysqlUser, "user", "wsjtx", "mysql username") + flag.StringVar(&mysqlUser, "user", "stationmonitor", "mysql username") flag.StringVar(&mysqlPass, "pass", "secret", "mysql password") flag.StringVar(&mysqlTable, "table", "wsjtx_all_txt", "mysql table name") flag.StringVar(&metricPath, "metricPath", "/metrics", "path for prometheus metric endpoint") @@ -101,13 +101,16 @@ func init() { func handleGetLastRequest(w http.ResponseWriter, r *http.Request) { - xstation := r.Header.Get("X-STATION") -// xproxystation := r.Header.Get("X-PROXY-STATION") - xidentifier := r.Header.Get("X-IDENTIFIER") - fmt.Printf("X-STATION: %s\n", xstation) - fmt.Printf("X-IDENTIFIER: %s\n", xidentifier) + xstation := r.Header.Get("X-Station") + xproxystation := r.Header.Get("X-Proxy-Station") + xidentifier := r.Header.Get("X-Identifier") + fmt.Printf("X-Station: %s\n", xstation) + fmt.Printf("X-Identifier: %s\n", xidentifier) // FIXME check for existence! and compare X-STATION against X-PROXY-STATION! + if xproxystation != xstation { + log.Errorf("ehmm... user and webserver have different opinion on station X-Station: , X-Proxy-Station:", xstation, xproxystation) + } last, err := getLast(xstation, xidentifier) if err != nil { @@ -163,11 +166,14 @@ func lineRequestHandler() chan LineRequest { func (lrm *LineRequestManager) handleLinesRequest(w http.ResponseWriter, r *http.Request) { var l Lines - xstation := r.Header.Get("X-STATION") -// xproxystation := r.Header.Get("X-PROXY-STATION") - xidentifier := r.Header.Get("X-IDENTIFIER") + xstation := r.Header.Get("X-Station") + xproxystation := r.Header.Get("X-Proxy-Station") + xidentifier := r.Header.Get("X-Identifier") // FIXME check for existence! and compare X-STATION against X-PROXY-STATION! + if xproxystation != xstation { + log.Errorf("ehmm... user and webserver have different opinion on station X-Station: , X-Proxy-Station:", xstation, xproxystation) + } b, err := ioutil.ReadAll(r.Body) defer r.Body.Close() diff --git a/cmd/http-exporter/mysql.go b/cmd/wsjtx-exporter/mysql.go similarity index 57% rename from cmd/http-exporter/mysql.go rename to cmd/wsjtx-exporter/mysql.go index 8bd0fe1..b48b1f3 100644 --- a/cmd/http-exporter/mysql.go +++ b/cmd/wsjtx-exporter/mysql.go @@ -18,9 +18,7 @@ func getLast(station string, identifier string) (LastTs, error) { } defer db.Close() - log.Info("starting query...") err := db.QueryRow("SELECT UNIX_TIMESTAMP(ts) FROM wsjtx_all_txt WHERE station = ? AND identifier = ? ORDER BY ts DESC LIMIT 1", station, identifier).Scan(&result.Last) - log.Info("finished query...") if err != nil { // FIXME if err.Error() == "sql: no rows in result set" { @@ -51,15 +49,12 @@ func handleMysql(result wsjtx.Result) { if err != nil { log.WithFields(log.Fields{"warning":err.Error()}).Warn("error while executing prepared statement..") - } else { - log.Info("insert done! :)") } } func init_db() { var cnt int - log.Debug("init_db()") db, _ := dbConn() row := db.QueryRow("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_NAME LIKE '%" + mysqlTable + "%';") @@ -74,21 +69,44 @@ func init_db() { } if cnt < 1 { - qry := "CREATE TABLE IF NOT EXISTS " + mysqlTable + " (" + - "ts timestamp NOT NULL," + - "station VARCHAR(16) NOT NULL," + - "callsign VARCHAR(16) NOT NULL," + - "band VARCHAR(10) NOT NULL," + - "continent VARCHAR(32) NOT NULL," + - "mode VARCHAR(16) NOT NULL," + - "dxcc VARCHAR(128) NOT NULL," + - "geohash VARCHAR(16) NOT NULL," + - "report TINYINT NOT NULL," + - "cqzone INT NOT NULL," + - "ituzone INT NOT NULL," + - "rx TINYINT NOT NULL," + - "PRIMARY KEY UC_" + mysqlTable + "(ts, station, callsign)," + - "INDEX idx_dxcc (dxcc));" + qry := "CREATE TABLE wsjtx_all_txt (" + + "ts datetime NOT NULL," + + "station varchar(16) NOT NULL," + + "callsign varchar(16) NOT NULL," + + "band varchar(10) NOT NULL," + + "continent varchar(32) NOT NULL," + + "mode varchar(16) NOT NULL," + + "dxcc varchar(128) NOT NULL," + + "geohash varchar(16) NOT NULL," + + "report tinyint(4) NOT NULL," + + "cqzone int(11) NOT NULL," + + "ituzone int(11) NOT NULL," + + "rx tinyint(4) NOT NULL," + + "identifier varchar(32) DEFAULT NULL," + + "PRIMARY KEY (ts,station,callsign)," + + "KEY idx_dxcc (dxcc)," + + "KEY idx_continent (continent)," + + "KEY idx_station (station)," + + "KEY idx_band (band)," + + "KEY idx_station_identifier (station,identifier)," + + "KEY idx_ts_station_band (ts,station,band)," + + "KEY idx_ts (ts));"; + +// qry := "CREATE TABLE IF NOT EXISTS " + mysqlTable + " (" + +// "ts timestamp NOT NULL," + +// "station VARCHAR(16) NOT NULL," + +// "callsign VARCHAR(16) NOT NULL," + +// "band VARCHAR(10) NOT NULL," + +// "continent VARCHAR(32) NOT NULL," + +// "mode VARCHAR(16) NOT NULL," + +// "dxcc VARCHAR(128) NOT NULL," + +// "geohash VARCHAR(16) NOT NULL," + +// "report TINYINT NOT NULL," + +// "cqzone INT NOT NULL," + +// "ituzone INT NOT NULL," + +// "rx TINYINT NOT NULL," + +// "PRIMARY KEY UC_" + mysqlTable + "(ts, station, callsign)," + +// "INDEX idx_dxcc (dxcc));" log.WithFields(log.Fields{"query":qry}).Debug("creating database..") _, err := db.Exec(qry) if err != nil { @@ -103,7 +121,9 @@ func init_db() { func dbConn() (db *sql.DB, err bool){ //db, er := sql.Open("mysql", mysql_user + ":" + mysql_pass + "@tcp(" + mysql_host + ")/" + mysql_db + "?parseTime=true&time_zone=%27UTC%27") - db, er := sql.Open("mysql", mysqlUser + ":" + mysqlPass + "@tcp(" + mysqlHost + ")/" + mysqlDb) + dbUri := mysqlUser + ":" + mysqlPass + "@tcp(" + mysqlHost + ")/" + mysqlDb + log.Tracef("dbUri: %s", dbUri) + db, er := sql.Open("mysql", dbUri) if er != nil { log.Error("db not reachable: %s",err) return nil, true @@ -111,7 +131,7 @@ func dbConn() (db *sql.DB, err bool){ pingerr := db.Ping() if pingerr != nil { - log.Error("db not pingable..") + log.Error("db not pingable: ",pingerr.Error()) return nil, true } diff --git a/cmd/http-exporter/prometheus.go b/cmd/wsjtx-exporter/prometheus.go similarity index 100% rename from cmd/http-exporter/prometheus.go rename to cmd/wsjtx-exporter/prometheus.go diff --git a/cmd/http-exporter/types.go b/cmd/wsjtx-exporter/types.go similarity index 100% rename from cmd/http-exporter/types.go rename to cmd/wsjtx-exporter/types.go diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..52f36df --- /dev/null +++ b/deploy.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +app="wsjtx-exporter" + +docker build -t 10.0.73.1:5000/$app:latest . || exit 1 +#export TAG=$(docker images|grep -v REPO|head -n1|awk '{print $3}') +#docker tag $TAG 10.0.73.1:5000/$app:latest || exit 1 +docker push 10.0.73.1:5000/$app:latest || exit 1 + +go get ./... +env GOOS=linux GOARCH=arm GOARM=5 go get ./... +env GOOS=windows GOARCH=amd64 go get ./... + +cp ~/go/bin/alltxt2http ~/Nextcloud/share/dl7le/binaries/linux64 +cp ~/go/bin/linux_arm/alltxt2http ~/Nextcloud/share/dl7le/binaries/raspi +cp ~/go/bin/windows_amd64/alltxt2http.exe ~/Nextcloud/share/dl7le/binaries/win64 diff --git a/wsjtx_all_txt.sql b/wsjtx_all_txt.sql new file mode 100644 index 0000000..e1c7a6d --- /dev/null +++ b/wsjtx_all_txt.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS wsjtx_all_txt ( + ts timestamp NOT NULL, + station VARCHAR(16) NOT NULL, + callsign VARCHAR(16) NOT NULL, + band VARCHAR(10) NOT NULL, + continent VARCHAR(32) NOT NULL, + mode VARCHAR(16) NOT NULL, + dxcc VARCHAR(128) NOT NULL, + geohash VARCHAR(16) NOT NULL, + report TINYINT NOT NULL, + cqzone INT NOT NULL, + ituzone INT NOT NULL, + rx TINYINT NOT NULL, + PRIMARY KEY PK_wsjtx_all_txt (ts, station, callsign), + INDEX idx_dxcc (dxcc) +);