package main import ( "fmt" "github.com/jnovack/flag" log "github.com/sirupsen/logrus" // "strings" // "strconv" "time" "os" "bytes" "net/http" "io/ioutil" "encoding/json" "bufio" // "bufio" // "runtime" // "sync" // "github.com/mmcloughlin/geohash" // "github.com/tzneal/ham-go/dxcc" // "github.com/denzs/wsjtx-dashboards/shared/httpstuff" "github.com/hpcloud/tail" ) var alltxt string var station string var identifier string var password string var uri string var minlen int var readall bool func usage() { fmt.Printf("Usage of %s:\n", os.Args[0]) flag.PrintDefaults() } func init() { flag.StringVar(&alltxt, "in", "", "path to wsjt-x ALL.TXT") flag.StringVar(&station, "station", "YOURCALL", "your station/username") flag.StringVar(&identifier, "identifier", "unset", "your identifier like location of rigname") flag.StringVar(&password, "password", "", "your password") flag.StringVar(&uri, "uri", "http://http-exporter", "uri to your http-exporter") flag.IntVar(&minlen, "minlen", 16, "minimal length for line to be pushed") flag.BoolVar(&readall, "readall", false, "submit whole file if nothing found at server") flag.Parse() if alltxt == "" || station == "" || identifier == "" || uri == "" { usage() // on windows could do some fancy dialog magic to ask for parameters and service installation ;) os.Exit(1) } formatter := &log.TextFormatter{ FullTimestamp: true, } log.SetFormatter(formatter) } func getTimestampFromDb() (int64, bool) { log.Info("doing timestamp stuff..") var ts LastTs client := http.Client{ Timeout: time.Second * 2, // Timeout after 2 seconds } req, err := http.NewRequest(http.MethodGet, uri + "/timestamp/getLast?identifier=" + identifier, nil) log.Info("done req..") if err != nil { log.Fatal(err) } req.Header.Set("X-STATION", station) req.Header.Set("X-IDENTIFIER", identifier) log.Info("done Headet.Set") res, getErr := client.Do(req) log.Info("done client.Do") if getErr != nil { log.Fatal(getErr) } if res.StatusCode != http.StatusOK { if res.StatusCode == 404 { // FIXME on windows show message to upload all.txt log.Info("Yeah, this seems to be our first contact with the server, as it doesnt have any records for %s:%s", station, identifier) return 0, false } fmt.Println("Non-OK HTTP status:", res.StatusCode) log.Panic("uh well, there seems to be some serious shit going on...") } if res.Body != nil { defer res.Body.Close() } body, readErr := ioutil.ReadAll(res.Body) if readErr != nil { log.Fatal(readErr) } log.Info("doing json stuff") jsonErr := json.Unmarshal(body, &ts) log.Info("done json stuff") if jsonErr != nil { log.Fatal(jsonErr) } log.Info("doing timestamp stuff done") return ts.Last, true } // https://forum.golangbridge.org/t/how-to-find-the-offset-of-a-byte-in-a-large-binary-file/16457/2 ;) // ScanFile returns the offset of the first occurence of a []byte in a file, // or -1 if []byte was not found in file, and seeks to the beginning of the searched []byte func findPosition(search []byte) (int64, bool) { f, err := os.Open(alltxt) if err != nil { log.Fatal(err) } ix := 0 r := bufio.NewReader(f) offset := int64(0) for ix < len(search) { b, err := r.ReadByte() if err != nil { return 0, false } if search[ix] == b { ix++ } else { ix = 0 } offset++ } // f.Seek(offset - int64(len(search)), 0 ) // Seeks to the beginning of the searched []byte //r.Close() //readseek.Close() f.Close() return offset - int64(len(search)), true } func unix2wsjtx(ts int64) (string) { t := time.Unix(ts, 0) s := t.UTC().Format("060102_150405") return s } func postLine(batchmode bool, lines chan string) { // FIXME implement batchmode // var l httpstuff.Lines for { select { case line := <- lines : l := Lines{} l.Text = append(l.Text, line) log.Infof("line: %s", l.Text) //var jsonStr = []byte(`{"title":"Buy cheese and bread for breakfast."}`) jsonStr, err := json.Marshal(l) if err != nil { log.Errorf("cant marshal json: ", err.Error()) } req, err := http.NewRequest("POST", uri + "/lines/insert", bytes.NewBuffer(jsonStr)) if err != nil { log.Errorf("crazy shit during newrequest", err.Error()) } req.Header.Set("X-STATION", station) req.Header.Set("X-IDENTIFIER", identifier) req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) if err != nil { log.Errorf("crazy shit during client", err.Error()) } defer resp.Body.Close() log.Trace("response Status: ", resp.Status) log.Trace("response Headers: ", resp.Header) body, _ := ioutil.ReadAll(resp.Body) log.Trace("response Body: ", string(body)) } } log.Trace("goodbye from postLine") } func readFile(offset int64, whence int, done chan bool) { log.Trace("start readFile %s", alltxt) lines := make(chan string ,2) go postLine((whence == 0), lines) go postLine((whence == 0), lines) t, _:= tail.TailFile(alltxt, tail.Config{Follow: true, Location: &tail.SeekInfo{Offset: offset, Whence: whence},}) for line := range t.Lines { // FIXME // at least minlen // check for logline format! lines <- line.Text } close(lines) log.Trace("readfile is finished") done <- true } func main(){ var lastDbTime int64 var foundts bool var offset int64 var whence int var foundoffset bool log.Info("starting up..") offset = 0 if !readall { whence = 2 lastDbTime, foundts = getTimestampFromDb() if foundts { log.Infof("last timestamp in db: %d", lastDbTime) wt := unix2wsjtx(lastDbTime) log.Infof("searching %s for: %s",alltxt, wt) offset, foundoffset = findPosition([]byte(wt)) if foundoffset { log.Trace("found %s at offset %d ", wt, offset) whence = 0 } else { log.Warnf("timestamp %s NOT found in %s! have you deleted your ALL.TXT or used an old identifier?", wt, alltxt) log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt) } } else { log.Warn("api has no data for %s:%s", station, identifier) log.Warn("sending spots from now on.. consider sending your %s to the database administrator for import", alltxt) } } else { log.Info("readall mode enabled..") whence = 0 } log.Infof("offset: %d whence: %d", offset, whence) done := make(chan bool) go readFile(offset, whence, done) <- done log.Info("..done") }