From c2855e7ded35b495aea2d9c9f38afc84c96287ee Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 24 May 2015 02:05:44 -0600 Subject: [PATCH 01/14] Implement decimation, except in R900 decoder. --- decode/decode.go | 81 +++++++++++++++++++++++++++++++++++------------- flags.go | 5 ++- idm/idm.go | 26 +++------------- r900/r900.go | 22 ++----------- recv.go | 6 ++-- scm/scm.go | 21 ++----------- 6 files changed, 76 insertions(+), 85 deletions(-) diff --git a/decode/decode.go b/decode/decode.go index 29bbb616d..9db50d211 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -25,19 +25,36 @@ import ( // PacketConfig specifies packet-specific radio configuration. type PacketConfig struct { - DataRate int + DataRate int + BlockSize, BlockSize2 int SymbolLength, SymbolLength2 int SampleRate int PreambleSymbols, PacketSymbols int PreambleLength, PacketLength int - BufferLength int Preamble string + BufferLength int + CenterFreq uint32 } +func (cfg PacketConfig) Decimate(decimation int) PacketConfig { + cfg.BlockSize /= decimation + cfg.BlockSize2 /= decimation + cfg.SymbolLength /= decimation + cfg.SymbolLength2 /= decimation + cfg.SampleRate /= decimation + + cfg.PreambleLength /= decimation + cfg.PacketLength /= decimation + + cfg.BufferLength /= decimation + + return cfg +} + func (cfg PacketConfig) Log() { log.Println("BlockSize:", cfg.BlockSize) log.Println("CenterFreq:", cfg.CenterFreq) @@ -55,6 +72,9 @@ func (cfg PacketConfig) Log() { type Decoder struct { Cfg PacketConfig + Decimation int + DecCfg PacketConfig + IQ []byte Signal []float64 Filtered []float64 @@ -70,16 +90,32 @@ type Decoder struct { } // Create a new decoder with the given packet configuration. -func NewDecoder(cfg PacketConfig, fastMag bool) (d Decoder) { +func NewDecoder(cfg PacketConfig, decimation int, fastMag bool) (d Decoder) { d.Cfg = cfg + d.Cfg.SymbolLength2 = d.Cfg.SymbolLength << 1 + d.Cfg.SampleRate = d.Cfg.DataRate * d.Cfg.SymbolLength + + d.Cfg.PreambleLength = d.Cfg.PreambleSymbols * d.Cfg.SymbolLength2 + d.Cfg.PacketLength = d.Cfg.PacketSymbols * d.Cfg.SymbolLength2 + + d.Cfg.BlockSize = NextPowerOf2(d.Cfg.PreambleLength) + d.Cfg.BlockSize2 = d.Cfg.BlockSize << 1 + + d.Cfg.BufferLength = d.Cfg.PacketLength + d.Cfg.BlockSize + + d.Decimation = decimation + d.DecCfg = d.Cfg.Decimate(d.Decimation) + fmt.Printf("%+v\n", d.Cfg) + fmt.Printf("%+v\n", d.DecCfg) + // Allocate necessary buffers. d.IQ = make([]byte, d.Cfg.BufferLength<<1) d.Signal = make([]float64, d.Cfg.BufferLength) - d.Filtered = make([]float64, d.Cfg.BufferLength) - d.Quantized = make([]byte, d.Cfg.BufferLength) + d.Filtered = make([]float64, d.DecCfg.BufferLength) + d.Quantized = make([]byte, d.DecCfg.BufferLength) - d.csum = make([]float64, d.Cfg.BlockSize+d.Cfg.SymbolLength2+1) + d.csum = make([]float64, (d.Cfg.BlockSize + d.Cfg.SymbolLength2 + 1)) // Calculate magnitude lookup table specified by -fastmag flag. if fastMag { @@ -99,10 +135,10 @@ func NewDecoder(cfg PacketConfig, fastMag bool) (d Decoder) { // Slice quantized sample buffer to make searching for the preamble more // memory local. Pre-allocate a flat buffer so memory is contiguous and // assign slices to the buffer. - d.slices = make([][]byte, d.Cfg.SymbolLength2) - flat := make([]byte, d.Cfg.BlockSize2-(d.Cfg.BlockSize2%d.Cfg.SymbolLength2)) + d.slices = make([][]byte, d.DecCfg.SymbolLength2) + flat := make([]byte, d.DecCfg.BlockSize2-(d.DecCfg.BlockSize2%d.DecCfg.SymbolLength2)) - symbolsPerBlock := d.Cfg.BlockSize2 / d.Cfg.SymbolLength2 + symbolsPerBlock := d.DecCfg.BlockSize2 / d.DecCfg.SymbolLength2 for symbolOffset := range d.slices { lower := symbolOffset * symbolsPerBlock upper := (symbolOffset + 1) * symbolsPerBlock @@ -111,7 +147,7 @@ func NewDecoder(cfg PacketConfig, fastMag bool) (d Decoder) { // Signal up to the final stage is 1-bit per byte. Allocate a buffer to // store packed version 8-bits per byte. - d.pkt = make([]byte, (d.Cfg.PacketSymbols+7)>>3) + d.pkt = make([]byte, (d.DecCfg.PacketSymbols+7)>>3) return } @@ -121,8 +157,8 @@ func (d Decoder) Decode(input []byte) []int { // Shift buffers to append new block. copy(d.IQ, d.IQ[d.Cfg.BlockSize<<1:]) copy(d.Signal, d.Signal[d.Cfg.BlockSize:]) - copy(d.Filtered, d.Filtered[d.Cfg.BlockSize:]) - copy(d.Quantized, d.Quantized[d.Cfg.BlockSize:]) + copy(d.Filtered, d.Filtered[d.DecCfg.BlockSize:]) + copy(d.Quantized, d.Quantized[d.DecCfg.BlockSize:]) copy(d.IQ[d.Cfg.PacketLength<<1:], input[:]) iqBlock := d.IQ[d.Cfg.PacketLength<<1:] @@ -132,16 +168,16 @@ func (d Decoder) Decode(input []byte) []int { d.demod.Execute(iqBlock, signalBlock) signalBlock = d.Signal[d.Cfg.PacketLength-d.Cfg.SymbolLength2:] - filterBlock := d.Filtered[d.Cfg.PacketLength-d.Cfg.SymbolLength2:] + filterBlock := d.Filtered[d.DecCfg.PacketLength-d.DecCfg.SymbolLength2:] // Perform matched filter on new block. d.Filter(signalBlock, filterBlock) // Perform bit-decision on new block. - Quantize(filterBlock, d.Quantized[d.Cfg.PacketLength-d.Cfg.SymbolLength2:]) + Quantize(filterBlock, d.Quantized[d.DecCfg.PacketLength-d.DecCfg.SymbolLength2:]) // Pack the quantized signal into slices for searching. - d.Pack(d.Quantized[:d.Cfg.BlockSize2], d.slices) + d.Pack(d.Quantized[:d.DecCfg.BlockSize2], d.slices) // Return a list of indexes the preamble exists at. return d.Search(d.slices, d.preamble) @@ -217,8 +253,9 @@ func (d Decoder) Filter(input, output []float64) { // Filter result is difference of summation of lower and upper symbols. lower := d.csum[d.Cfg.SymbolLength:] upper := d.csum[d.Cfg.SymbolLength2:] - for idx := range input[:len(input)-d.Cfg.SymbolLength2] { - output[idx] = (lower[idx] - d.csum[idx]) - (upper[idx] - lower[idx]) + n := len(input) - d.Cfg.SymbolLength2 + for idx := 0; idx < n; idx += d.Decimation { + output[idx/d.Decimation] = (lower[idx] - d.csum[idx]) - (upper[idx] - lower[idx]) } return @@ -239,7 +276,7 @@ func Quantize(input []float64, output []byte) { func (d Decoder) Pack(input []byte, slices [][]byte) { for symbolOffset, slice := range slices { for symbolIdx := range slice { - slice[symbolIdx] = input[symbolIdx*d.Cfg.SymbolLength2+symbolOffset] + slice[symbolIdx] = input[symbolIdx*d.DecCfg.SymbolLength2+symbolOffset] } } @@ -254,7 +291,7 @@ func (d Decoder) Search(slices [][]byte, preamble []byte) (indexes []int) { for symbolOffset, slice := range slices { for symbolIdx := range slice[:len(slice)-preambleLength] { if bytes.Equal(preamble, slice[symbolIdx:][:preambleLength]) { - indexes = append(indexes, symbolIdx*d.Cfg.SymbolLength2+symbolOffset) + indexes = append(indexes, symbolIdx*d.DecCfg.SymbolLength2+symbolOffset) } } } @@ -274,14 +311,14 @@ func (d Decoder) Slice(indices []int) (pkts [][]byte) { for _, qIdx := range indices { // Check that we're still within the first sample block. We'll catch // the message on the next sample block otherwise. - if qIdx > d.Cfg.BlockSize { + if qIdx > d.DecCfg.BlockSize { continue } // Packet is 1 bit per byte, pack to 8-bits per byte. - for pIdx := 0; pIdx < d.Cfg.PacketSymbols; pIdx++ { + for pIdx := 0; pIdx < d.DecCfg.PacketSymbols; pIdx++ { d.pkt[pIdx>>3] <<= 1 - d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.Cfg.SymbolLength2)] + d.pkt[pIdx>>3] |= d.Quantized[qIdx+(pIdx*d.DecCfg.SymbolLength2)] } // Store the packet in the seen map and append to the packet list. diff --git a/flags.go b/flags.go index e1aac4b83..7259a3ab8 100644 --- a/flags.go +++ b/flags.go @@ -39,7 +39,9 @@ var sampleFile *os.File var msgType = flag.String("msgtype", "scm", "message type to receive: scm, idm or r900") var fastMag = flag.Bool("fastmag", false, "use faster alpha max + beta min magnitude approximation") -var symbolLength = flag.Int("symbollength", 73, "symbol length in samples, see -help for valid lengths") +var symbolLength = flag.Int("symbollength", 72, "symbol length in samples, see -help for valid lengths") + +var decimation = flag.Int("decimation", 1, "integer decimation factor, keep every nth sample") var timeLimit = flag.Duration("duration", 0, "time to run for, 0 for infinite, ex. 1h5m10s") var meterID UintMap @@ -64,6 +66,7 @@ func RegisterFlags() { "samplefile": true, "msgtype": true, "symbollength": true, + "decimation": true, "duration": true, "filterid": true, "filtertype": true, diff --git a/idm/idm.go b/idm/idm.go index 5fa7fea56..1cb124d43 100644 --- a/idm/idm.go +++ b/idm/idm.go @@ -28,27 +28,13 @@ import ( ) func NewPacketConfig(symbolLength int) (cfg decode.PacketConfig) { - cfg.DataRate = 32768 - cfg.CenterFreq = 912600155 - + cfg.DataRate = 32768 cfg.SymbolLength = symbolLength - cfg.SymbolLength2 = cfg.SymbolLength << 1 - - cfg.SampleRate = cfg.DataRate * cfg.SymbolLength - cfg.PreambleSymbols = 32 cfg.PacketSymbols = 92 * 8 - - cfg.PreambleLength = cfg.PreambleSymbols * cfg.SymbolLength2 - cfg.PacketLength = cfg.PacketSymbols * cfg.SymbolLength2 - - cfg.BlockSize = decode.NextPowerOf2(cfg.PreambleLength) - cfg.BlockSize2 = cfg.BlockSize << 1 - - cfg.BufferLength = cfg.PacketLength + cfg.BlockSize - cfg.Preamble = "01010101010101010001011010100011" + return } @@ -65,8 +51,8 @@ func (p Parser) Cfg() decode.PacketConfig { return p.Decoder.Cfg } -func NewParser(symbolLength int, fastMag bool) (p Parser) { - p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), fastMag) +func NewParser(symbolLength, decimation int, fastMag bool) (p Parser) { + p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), decimation, fastMag) p.CRC = crc.NewCRC("CCITT", 0xFFFF, 0x1021, 0x1D0F) return } @@ -94,10 +80,6 @@ type IDM struct { type Interval [47]uint16 -// func (interval Interval) MarshalText() (text []byte, err error) { -// return []byte(fmt.Sprintf("%+v", interval)), nil -// } - func (interval Interval) Record() (r []string) { for _, val := range interval { r = append(r, strconv.FormatUint(uint64(val), 10)) diff --git a/r900/r900.go b/r900/r900.go index 8aa5c4c57..c756e0908 100644 --- a/r900/r900.go +++ b/r900/r900.go @@ -32,26 +32,10 @@ const ( ) func NewPacketConfig(symbolLength int) (cfg decode.PacketConfig) { - cfg.DataRate = 32768 - cfg.CenterFreq = 912380000 - - cfg.SymbolLength = symbolLength - cfg.SymbolLength2 = cfg.SymbolLength << 1 - - cfg.SampleRate = cfg.DataRate * cfg.SymbolLength - + cfg.DataRate = 32768 cfg.PreambleSymbols = 32 cfg.PacketSymbols = 116 - - cfg.PreambleLength = cfg.PreambleSymbols * cfg.SymbolLength2 - cfg.PacketLength = cfg.PacketSymbols * cfg.SymbolLength2 - - cfg.BlockSize = decode.NextPowerOf2(cfg.PreambleLength) - cfg.BlockSize2 = cfg.BlockSize << 1 - - cfg.BufferLength = cfg.PacketLength + cfg.BlockSize - cfg.Preamble = "00000000000000001110010101100100" return @@ -67,8 +51,8 @@ type Parser struct { quantized []byte } -func NewParser(symbolLength int, fastMag bool) (p Parser) { - p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), fastMag) +func NewParser(symbolLength, decimation int, fastMag bool) (p Parser) { + p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), 1, fastMag) // GF of order 32, polynomial 37, generator 2. p.field = gf.NewField(32, 37, 2) diff --git a/recv.go b/recv.go index c550e8c08..ac48fcdac 100644 --- a/recv.go +++ b/recv.go @@ -44,11 +44,11 @@ type Receiver struct { func (rcvr *Receiver) NewReceiver() { switch strings.ToLower(*msgType) { case "scm": - rcvr.p = scm.NewParser(*symbolLength, *fastMag) + rcvr.p = scm.NewParser(*symbolLength, *decimation, *fastMag) case "idm": - rcvr.p = idm.NewParser(*symbolLength, *fastMag) + rcvr.p = idm.NewParser(*symbolLength, *decimation, *fastMag) case "r900": - rcvr.p = r900.NewParser(*symbolLength, *fastMag) + rcvr.p = r900.NewParser(*symbolLength, *decimation, *fastMag) default: log.Fatalf("Invalid message type: %q\n", *msgType) } diff --git a/scm/scm.go b/scm/scm.go index fd8122604..5d8ccb620 100644 --- a/scm/scm.go +++ b/scm/scm.go @@ -26,26 +26,11 @@ import ( ) func NewPacketConfig(symbolLength int) (cfg decode.PacketConfig) { - cfg.DataRate = 32768 - cfg.CenterFreq = 912600155 - + cfg.DataRate = 32768 cfg.SymbolLength = symbolLength - cfg.SymbolLength2 = cfg.SymbolLength << 1 - - cfg.SampleRate = cfg.DataRate * cfg.SymbolLength - cfg.PreambleSymbols = 21 cfg.PacketSymbols = 96 - - cfg.PreambleLength = cfg.PreambleSymbols * cfg.SymbolLength2 - cfg.PacketLength = cfg.PacketSymbols * cfg.SymbolLength2 - - cfg.BlockSize = decode.NextPowerOf2(cfg.PreambleLength) - cfg.BlockSize2 = cfg.BlockSize << 1 - - cfg.BufferLength = cfg.PacketLength + cfg.BlockSize - cfg.Preamble = "111110010101001100000" return @@ -56,8 +41,8 @@ type Parser struct { crc.CRC } -func NewParser(symbolLength int, fastMag bool) (p Parser) { - p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), fastMag) +func NewParser(symbolLength, decimation int, fastMag bool) (p Parser) { + p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), decimation, fastMag) p.CRC = crc.NewCRC("BCH", 0, 0x6F63, 0) return } From f7828796bfb16f0102f83afdd18dd62c63154d32 Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 24 May 2015 02:18:06 -0600 Subject: [PATCH 02/14] Remove config debugging statements. --- decode/decode.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/decode/decode.go b/decode/decode.go index 9db50d211..60b2ce06b 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -106,8 +106,6 @@ func NewDecoder(cfg PacketConfig, decimation int, fastMag bool) (d Decoder) { d.Decimation = decimation d.DecCfg = d.Cfg.Decimate(d.Decimation) - fmt.Printf("%+v\n", d.Cfg) - fmt.Printf("%+v\n", d.DecCfg) // Allocate necessary buffers. d.IQ = make([]byte, d.Cfg.BufferLength<<1) From 588ef5220ce30ce685ff823f8e31e79cb04f5974 Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 24 May 2015 02:24:13 -0600 Subject: [PATCH 03/14] Read whole blocks from server concurrently with io.Pipe(). --- recv.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/recv.go b/recv.go index ac48fcdac..801affc35 100644 --- a/recv.go +++ b/recv.go @@ -20,6 +20,7 @@ import ( "encoding/xml" "flag" "fmt" + "io" "log" "os" "os/signal" @@ -111,6 +112,19 @@ func (rcvr *Receiver) Run() { tLimit = time.After(*timeLimit) } + in, out := io.Pipe() + + go func() { + tcpBlock := make([]byte, 16384) + for { + n, err := rcvr.Read(tcpBlock) + if err != nil { + return + } + out.Write(tcpBlock[:n]) + } + }() + block := make([]byte, rcvr.p.Cfg().BlockSize2) start := time.Now() @@ -124,7 +138,7 @@ func (rcvr *Receiver) Run() { return default: // Read new sample block. - _, err := rcvr.Read(block) + _, err := in.Read(block) if err != nil { log.Fatal("Error reading samples: ", err) } From 1373ceb5d744ee345b9aeb4a253806d89b343b7a Mon Sep 17 00:00:00 2001 From: bemasher Date: Mon, 25 May 2015 01:43:20 -0600 Subject: [PATCH 04/14] Fix block read, reads might span writes to the pipe. --- recv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recv.go b/recv.go index 801affc35..bf4d6dcd3 100644 --- a/recv.go +++ b/recv.go @@ -138,7 +138,7 @@ func (rcvr *Receiver) Run() { return default: // Read new sample block. - _, err := in.Read(block) + _, err := io.ReadFull(in, block) if err != nil { log.Fatal("Error reading samples: ", err) } From 586359c74afc939ae639d30b0b4c0304e22b692f Mon Sep 17 00:00:00 2001 From: bemasher Date: Mon, 25 May 2015 02:21:33 -0600 Subject: [PATCH 05/14] Move decimation to demodulator, only MagLUT for now. --- decode/decode.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/decode/decode.go b/decode/decode.go index 60b2ce06b..072a3e2dc 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -109,11 +109,11 @@ func NewDecoder(cfg PacketConfig, decimation int, fastMag bool) (d Decoder) { // Allocate necessary buffers. d.IQ = make([]byte, d.Cfg.BufferLength<<1) - d.Signal = make([]float64, d.Cfg.BufferLength) + d.Signal = make([]float64, d.DecCfg.BufferLength) d.Filtered = make([]float64, d.DecCfg.BufferLength) d.Quantized = make([]byte, d.DecCfg.BufferLength) - d.csum = make([]float64, (d.Cfg.BlockSize + d.Cfg.SymbolLength2 + 1)) + d.csum = make([]float64, (d.DecCfg.BlockSize + d.DecCfg.SymbolLength2 + 1)) // Calculate magnitude lookup table specified by -fastmag flag. if fastMag { @@ -154,18 +154,18 @@ func NewDecoder(cfg PacketConfig, decimation int, fastMag bool) (d Decoder) { func (d Decoder) Decode(input []byte) []int { // Shift buffers to append new block. copy(d.IQ, d.IQ[d.Cfg.BlockSize<<1:]) - copy(d.Signal, d.Signal[d.Cfg.BlockSize:]) + copy(d.Signal, d.Signal[d.DecCfg.BlockSize:]) copy(d.Filtered, d.Filtered[d.DecCfg.BlockSize:]) copy(d.Quantized, d.Quantized[d.DecCfg.BlockSize:]) copy(d.IQ[d.Cfg.PacketLength<<1:], input[:]) iqBlock := d.IQ[d.Cfg.PacketLength<<1:] - signalBlock := d.Signal[d.Cfg.PacketLength:] + signalBlock := d.Signal[d.DecCfg.PacketLength:] // Compute the magnitude of the new block. d.demod.Execute(iqBlock, signalBlock) - signalBlock = d.Signal[d.Cfg.PacketLength-d.Cfg.SymbolLength2:] + signalBlock = d.Signal[d.DecCfg.PacketLength-d.DecCfg.SymbolLength2:] filterBlock := d.Filtered[d.DecCfg.PacketLength-d.DecCfg.SymbolLength2:] // Perform matched filter on new block. @@ -202,8 +202,13 @@ func NewSqrtMagLUT() (lut MagLUT) { // Calculates complex magnitude on given IQ stream writing result to output. func (lut MagLUT) Execute(input []byte, output []float64) { - for idx := 0; idx < len(input); idx += 2 { - output[idx>>1] = math.Sqrt(lut[input[idx]] + lut[input[idx+1]]) + decIdx := 0 + dec := (len(input) / len(output)) + + for idx := 0; decIdx < len(output); idx += dec { + // fmt.Println(idx, idx/dec, len(input), len(output)) + output[decIdx] = math.Sqrt(lut[input[idx]] + lut[input[idx+1]]) + decIdx++ } } @@ -249,11 +254,11 @@ func (d Decoder) Filter(input, output []float64) { } // Filter result is difference of summation of lower and upper symbols. - lower := d.csum[d.Cfg.SymbolLength:] - upper := d.csum[d.Cfg.SymbolLength2:] - n := len(input) - d.Cfg.SymbolLength2 - for idx := 0; idx < n; idx += d.Decimation { - output[idx/d.Decimation] = (lower[idx] - d.csum[idx]) - (upper[idx] - lower[idx]) + lower := d.csum[d.DecCfg.SymbolLength:] + upper := d.csum[d.DecCfg.SymbolLength2:] + n := len(input) - d.DecCfg.SymbolLength2 + for idx := 0; idx < n; idx++ { + output[idx] = (lower[idx] - d.csum[idx]) - (upper[idx] - lower[idx]) } return From 3a29e9b7ff05b9ba3ff71172aaf93c54e7231942 Mon Sep 17 00:00:00 2001 From: bemasher Date: Thu, 28 May 2015 21:44:59 -0600 Subject: [PATCH 06/14] Implement decimation in R900 decoder. --- r900/r900.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/r900/r900.go b/r900/r900.go index c756e0908..a34762ec7 100644 --- a/r900/r900.go +++ b/r900/r900.go @@ -34,6 +34,7 @@ const ( func NewPacketConfig(symbolLength int) (cfg decode.PacketConfig) { cfg.CenterFreq = 912380000 cfg.DataRate = 32768 + cfg.SymbolLength = symbolLength cfg.PreambleSymbols = 32 cfg.PacketSymbols = 116 cfg.Preamble = "00000000000000001110010101100100" @@ -52,14 +53,14 @@ type Parser struct { } func NewParser(symbolLength, decimation int, fastMag bool) (p Parser) { - p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), 1, fastMag) + p.Decoder = decode.NewDecoder(NewPacketConfig(symbolLength), decimation, fastMag) // GF of order 32, polynomial 37, generator 2. p.field = gf.NewField(32, 37, 2) - p.csum = make([]float64, p.Decoder.Cfg.BufferLength+1) - p.filtered = make([][3]float64, p.Decoder.Cfg.BufferLength) - p.quantized = make([]byte, p.Decoder.Cfg.BufferLength) + p.csum = make([]float64, p.Decoder.DecCfg.BufferLength+1) + p.filtered = make([][3]float64, p.Decoder.DecCfg.BufferLength) + p.quantized = make([]byte, p.Decoder.DecCfg.BufferLength) return } @@ -103,12 +104,12 @@ func (p Parser) Filter() { // This is basically unreadable because of a lot of algebraic // simplification but is necessary for efficiency. - for idx := 0; idx < p.Decoder.Cfg.BufferLength-p.Decoder.Cfg.SymbolLength*4; idx++ { + for idx := 0; idx < p.Decoder.DecCfg.BufferLength-p.Decoder.DecCfg.SymbolLength*4; idx++ { c0 := p.csum[idx] - c1 := p.csum[idx+p.Decoder.Cfg.SymbolLength] * 2 - c2 := p.csum[idx+p.Decoder.Cfg.SymbolLength*2] * 2 - c3 := p.csum[idx+p.Decoder.Cfg.SymbolLength*3] * 2 - c4 := p.csum[idx+p.Decoder.Cfg.SymbolLength*4] + c1 := p.csum[idx+p.Decoder.DecCfg.SymbolLength] * 2 + c2 := p.csum[idx+p.Decoder.DecCfg.SymbolLength*2] * 2 + c3 := p.csum[idx+p.Decoder.DecCfg.SymbolLength*3] * 2 + c4 := p.csum[idx+p.Decoder.DecCfg.SymbolLength*4] p.filtered[idx][0] = c2 - c4 - c0 // 1100 p.filtered[idx][1] = c1 - c2 + c3 - c4 - c0 // 1010 @@ -153,8 +154,8 @@ func (p Parser) Parse(indices []int) (msgs []parse.Message) { p.Filter() p.Quantize() - preambleLength := p.Decoder.Cfg.PreambleLength - symbolLength := p.Decoder.Cfg.SymbolLength + preambleLength := p.Decoder.DecCfg.PreambleLength + symbolLength := p.Decoder.DecCfg.SymbolLength symbols := make([]byte, 21) zeros := make([]byte, 5) @@ -162,13 +163,13 @@ func (p Parser) Parse(indices []int) (msgs []parse.Message) { seen := make(map[string]bool) for _, preambleIdx := range indices { - if preambleIdx > p.Decoder.Cfg.BlockSize { + if preambleIdx > p.Decoder.DecCfg.BlockSize { break } payloadIdx := preambleIdx + preambleLength var digits string - for idx := 0; idx < PayloadSymbols*4*p.Decoder.Cfg.SymbolLength; idx += symbolLength * 4 { + for idx := 0; idx < PayloadSymbols*4*p.Decoder.DecCfg.SymbolLength; idx += symbolLength * 4 { qIdx := payloadIdx + idx digits += strconv.Itoa(int(p.quantized[qIdx])) From efea18916ce9750420ab22aafa90f419becdad3d Mon Sep 17 00:00:00 2001 From: bemasher Date: Thu, 28 May 2015 21:55:18 -0600 Subject: [PATCH 07/14] Implement decimation in AlphaMaxBetaMinLUT. --- decode/decode.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/decode/decode.go b/decode/decode.go index 072a3e2dc..926a3e9c9 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -206,7 +206,6 @@ func (lut MagLUT) Execute(input []byte, output []float64) { dec := (len(input) / len(output)) for idx := 0; decIdx < len(output); idx += dec { - // fmt.Println(idx, idx/dec, len(input), len(output)) output[decIdx] = math.Sqrt(lut[input[idx]] + lut[input[idx+1]]) decIdx++ } @@ -231,14 +230,18 @@ func (lut AlphaMaxBetaMinLUT) Execute(input []byte, output []float64) { ß = 0.392699081699 ) - for idx := 0; idx < len(input); idx += 2 { + decIdx := 0 + dec := (len(input) / len(output)) + + for idx := 0; decIdx < len(output); idx += dec { i := lut[input[idx]] q := lut[input[idx+1]] if i > q { - output[idx>>1] = α*i + ß*q + output[decIdx] = α*i + ß*q } else { - output[idx>>1] = α*q + ß*i + output[decIdx] = α*q + ß*i } + decIdx++ } } From 3bd0a4e5583feb7e5f916969cd0d9ca2536b21ca Mon Sep 17 00:00:00 2001 From: bemasher Date: Thu, 28 May 2015 23:16:49 -0600 Subject: [PATCH 08/14] Update flags in README. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 449c0f401..e22cf0b2d 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Available command-line flags are as follows: ``` Usage of rtlamr: -cpuprofile=: write cpu profile to this file + -decimation=1: integer decimation factor, keep every nth sample -duration=0: time to run for, 0 for infinite, ex. 1h5m10s -fastmag=false: use faster alpha max + beta min magnitude approximation -filterid=: display only messages matching an id in a comma-separated list of ids. @@ -36,7 +37,7 @@ Usage of rtlamr: -quiet=false: suppress printing state information at startup -samplefile=/dev/null: raw signal dump file -single=false: one shot execution - -symbollength=73: symbol length in samples, see -help for valid lengths + -symbollength=72: symbol length in samples, see -help for valid lengths rtltcp specific: -agcmode=false: enable/disable rtl agc From 22906a5949bd4983d8c00fe219343140520e16ca Mon Sep 17 00:00:00 2001 From: bemasher Date: Thu, 28 May 2015 23:18:51 -0600 Subject: [PATCH 09/14] Update link in README to point to gh-pages documentation. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e22cf0b2d..a72999453 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ $ rtlamr If you want to run the spectrum server on a different machine than the receiver you'll want to specify an address to listen on that is accessible from the machine `rtlamr` will run on with the `-a` option for `rtl_tcp` with an address accessible by the system running the receiver. ### Messages -Currently both SCM (Standard Consumption Message) and IDM (Interval Data Message) packets can be decoded but are mutually exclusive, you cannot receive both simultaneously. See [Wikipedia: Encoder Receiver Transmitter](http://en.wikipedia.org/wiki/Encoder_receiver_transmitter) for more details on packet structure. +Currently both SCM (Standard Consumption Message) and IDM (Interval Data Message) packets can be decoded but are mutually exclusive, you cannot receive both simultaneously. See [RTLAMR: Protocol](http://bemasher.github.io/rtlamr/protocol.html) for more details on packet structure. ### Sensitivity Using a NooElec NESDR Nano R820T with the provided antenna, I can reliably receive standard consumption messages from ~300 different meters and intermittently from another ~600 meters. These figures are calculated from the number of messages received during a 25 minute window. Reliably in this case means receiving at least 10 of the expected 12 messages and intermittently means 3-9 messages. From 953f371bb6c59f5afbf25e42fbf80816d33e1280 Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 31 May 2015 04:21:05 -0600 Subject: [PATCH 10/14] Move R900 message statement. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a72999453..fa36c4ddc 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ If you want to run the spectrum server on a different machine than the receiver ### Messages Currently both SCM (Standard Consumption Message) and IDM (Interval Data Message) packets can be decoded but are mutually exclusive, you cannot receive both simultaneously. See [RTLAMR: Protocol](http://bemasher.github.io/rtlamr/protocol.html) for more details on packet structure. +There's now experimental support for meters with R900 transmitters! + ### Sensitivity Using a NooElec NESDR Nano R820T with the provided antenna, I can reliably receive standard consumption messages from ~300 different meters and intermittently from another ~600 meters. These figures are calculated from the number of messages received during a 25 minute window. Reliably in this case means receiving at least 10 of the expected 12 messages and intermittently means 3-9 messages. @@ -80,8 +82,6 @@ Check out the table of meters I've been compiling from various internet sources: If you've got a meter not on the list that you've successfully received messages from, you can submit this info via a form available at the link above. -There's now experimental support for meters with R900 transmitters! - ### Ethics _Do not use this for nefarious purposes._ If you do, I don't want to know about it, I am not and will not be responsible for your lack of common decency and/or foresight. However, if you find a clever non-evil use for this, by all means, share. From de4bffe446911fa2b87611d58341b98e9656bd46 Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 7 Jun 2015 00:15:48 -0600 Subject: [PATCH 11/14] Fix csum length. --- decode/decode.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/decode/decode.go b/decode/decode.go index 926a3e9c9..abdf41bf1 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -113,7 +113,7 @@ func NewDecoder(cfg PacketConfig, decimation int, fastMag bool) (d Decoder) { d.Filtered = make([]float64, d.DecCfg.BufferLength) d.Quantized = make([]byte, d.DecCfg.BufferLength) - d.csum = make([]float64, (d.DecCfg.BlockSize + d.DecCfg.SymbolLength2 + 1)) + d.csum = make([]float64, (d.DecCfg.PacketLength - d.DecCfg.SymbolLength2 + 1)) // Calculate magnitude lookup table specified by -fastmag flag. if fastMag { From 04d73e42a794f64caac5118366e4a0942b14c59a Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 7 Jun 2015 00:53:12 -0600 Subject: [PATCH 12/14] Update logging to include decimated configuration. --- decode/decode.go | 41 ++++++++++++++++++++++++++++++----------- parse/parse.go | 1 + recv.go | 2 +- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/decode/decode.go b/decode/decode.go index abdf41bf1..cc96cebe3 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -46,6 +46,7 @@ func (cfg PacketConfig) Decimate(decimation int) PacketConfig { cfg.SymbolLength /= decimation cfg.SymbolLength2 /= decimation cfg.SampleRate /= decimation + cfg.DataRate /= decimation cfg.PreambleLength /= decimation cfg.PacketLength /= decimation @@ -55,17 +56,35 @@ func (cfg PacketConfig) Decimate(decimation int) PacketConfig { return cfg } -func (cfg PacketConfig) Log() { - log.Println("BlockSize:", cfg.BlockSize) - log.Println("CenterFreq:", cfg.CenterFreq) - log.Println("SampleRate:", cfg.SampleRate) - log.Println("DataRate:", cfg.DataRate) - log.Println("SymbolLength:", cfg.SymbolLength) - log.Println("PreambleSymbols:", cfg.PreambleSymbols) - log.Println("PreambleLength:", cfg.PreambleLength) - log.Println("PacketSymbols:", cfg.PacketSymbols) - log.Println("PacketLength:", cfg.PacketLength) - log.Println("Preamble:", cfg.Preamble) +func (d Decoder) Log() { + if d.Decimation != 1 { + log.Printf("BlockSize: %d|%d\n", d.Cfg.BlockSize, d.DecCfg.BlockSize) + log.Println("CenterFreq:", d.Cfg.CenterFreq) + log.Printf("SampleRate: %d|%d\n", d.Cfg.SampleRate, d.DecCfg.SampleRate) + log.Printf("DataRate: %d|%d\n", d.Cfg.DataRate, d.DecCfg.DataRate) + log.Printf("SymbolLength: %d|%d\n", d.Cfg.SymbolLength, d.DecCfg.SymbolLength) + log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols) + log.Printf("PreambleLength: %d|%d\n", d.Cfg.PreambleLength, d.DecCfg.PreambleLength) + log.Println("PacketSymbols:", d.Cfg.PacketSymbols) + log.Printf("PacketLength: %d|%d\n", d.Cfg.PacketLength, d.DecCfg.PacketLength) + log.Println("Preamble:", d.Cfg.Preamble) + + if d.Cfg.SymbolLength%d.Decimation != 0 { + log.Println("Warning: decimation factor gives non-integer symbol length, sensitivity may be poor") + } + + return + } + + log.Println("CenterFreq:", d.Cfg.CenterFreq) + log.Println("SampleRate:", d.Cfg.SampleRate) + log.Println("DataRate:", d.Cfg.DataRate) + log.Println("SymbolLength:", d.Cfg.SymbolLength) + log.Println("PreambleSymbols:", d.Cfg.PreambleSymbols) + log.Println("PreambleLength:", d.Cfg.PreambleLength) + log.Println("PacketSymbols:", d.Cfg.PacketSymbols) + log.Println("PacketLength:", d.Cfg.PacketLength) + log.Println("Preamble:", d.Cfg.Preamble) } // Decoder contains buffers and radio configuration. diff --git a/parse/parse.go b/parse/parse.go index 0a2655729..4f719d12d 100644 --- a/parse/parse.go +++ b/parse/parse.go @@ -42,6 +42,7 @@ type Parser interface { Parse([]int) []Message Dec() decode.Decoder Cfg() decode.PacketConfig + Log() } type Message interface { diff --git a/recv.go b/recv.go index bf4d6dcd3..1aac532f5 100644 --- a/recv.go +++ b/recv.go @@ -55,7 +55,7 @@ func (rcvr *Receiver) NewReceiver() { } if !*quiet { - rcvr.p.Cfg().Log() + rcvr.p.Log() } // Connect to rtl_tcp server. From 8f3cb5205ec1f442e1e7b03bb9ad95bbf6fc40a5 Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 7 Jun 2015 01:24:38 -0600 Subject: [PATCH 13/14] Apply gofmt to everything. --- parse/parse.go | 156 ++++++++++++++++++++++++------------------------- 1 file changed, 78 insertions(+), 78 deletions(-) diff --git a/parse/parse.go b/parse/parse.go index 4f719d12d..4e4a2ce1c 100644 --- a/parse/parse.go +++ b/parse/parse.go @@ -1,78 +1,78 @@ -package parse - -import ( - "fmt" - "strconv" - "time" - - "github.com/bemasher/rtlamr/decode" - - "github.com/bemasher/rtlamr/csv" -) - -const ( - TimeFormat = "2006-01-02T15:04:05.000" -) - -type Data struct { - Bits string - Bytes []byte -} - -func NewDataFromBytes(data []byte) (d Data) { - d.Bytes = data - for _, b := range data { - d.Bits += fmt.Sprintf("%08b", b) - } - - return -} - -func NewDataFromBits(data string) (d Data) { - d.Bits = data - d.Bytes = make([]byte, (len(data)+7)>>3) - for idx := 0; idx < len(data); idx += 8 { - b, _ := strconv.ParseUint(d.Bits[idx:idx+8], 2, 8) - d.Bytes[idx>>3] = uint8(b) - } - return -} - -type Parser interface { - Parse([]int) []Message - Dec() decode.Decoder - Cfg() decode.PacketConfig - Log() -} - -type Message interface { - csv.Recorder - MsgType() string - MeterID() uint32 - MeterType() uint8 -} - -type LogMessage struct { - Time time.Time - Offset int64 - Length int - Message -} - -func (msg LogMessage) String() string { - return fmt.Sprintf("{Time:%s Offset:%d Length:%d %s:%s}", - msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.MsgType(), msg.Message, - ) -} - -func (msg LogMessage) StringNoOffset() string { - return fmt.Sprintf("{Time:%s %s:%s}", msg.Time.Format(TimeFormat), msg.MsgType(), msg.Message) -} - -func (msg LogMessage) Record() (r []string) { - r = append(r, msg.Time.Format(time.RFC3339Nano)) - r = append(r, strconv.FormatInt(msg.Offset, 10)) - r = append(r, strconv.FormatInt(int64(msg.Length), 10)) - r = append(r, msg.Message.Record()...) - return r -} +package parse + +import ( + "fmt" + "strconv" + "time" + + "github.com/bemasher/rtlamr/decode" + + "github.com/bemasher/rtlamr/csv" +) + +const ( + TimeFormat = "2006-01-02T15:04:05.000" +) + +type Data struct { + Bits string + Bytes []byte +} + +func NewDataFromBytes(data []byte) (d Data) { + d.Bytes = data + for _, b := range data { + d.Bits += fmt.Sprintf("%08b", b) + } + + return +} + +func NewDataFromBits(data string) (d Data) { + d.Bits = data + d.Bytes = make([]byte, (len(data)+7)>>3) + for idx := 0; idx < len(data); idx += 8 { + b, _ := strconv.ParseUint(d.Bits[idx:idx+8], 2, 8) + d.Bytes[idx>>3] = uint8(b) + } + return +} + +type Parser interface { + Parse([]int) []Message + Dec() decode.Decoder + Cfg() decode.PacketConfig + Log() +} + +type Message interface { + csv.Recorder + MsgType() string + MeterID() uint32 + MeterType() uint8 +} + +type LogMessage struct { + Time time.Time + Offset int64 + Length int + Message +} + +func (msg LogMessage) String() string { + return fmt.Sprintf("{Time:%s Offset:%d Length:%d %s:%s}", + msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.MsgType(), msg.Message, + ) +} + +func (msg LogMessage) StringNoOffset() string { + return fmt.Sprintf("{Time:%s %s:%s}", msg.Time.Format(TimeFormat), msg.MsgType(), msg.Message) +} + +func (msg LogMessage) Record() (r []string) { + r = append(r, msg.Time.Format(time.RFC3339Nano)) + r = append(r, strconv.FormatInt(msg.Offset, 10)) + r = append(r, strconv.FormatInt(int64(msg.Length), 10)) + r = append(r, msg.Message.Record()...) + return r +} From 7f05214d2bab9ce671fee288afd0b7ddc4b04834 Mon Sep 17 00:00:00 2001 From: bemasher Date: Sun, 7 Jun 2015 01:38:21 -0600 Subject: [PATCH 14/14] Add decimation factor checking. Clarify warning. --- decode/decode.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/decode/decode.go b/decode/decode.go index cc96cebe3..1c397e20f 100644 --- a/decode/decode.go +++ b/decode/decode.go @@ -70,7 +70,11 @@ func (d Decoder) Log() { log.Println("Preamble:", d.Cfg.Preamble) if d.Cfg.SymbolLength%d.Decimation != 0 { - log.Println("Warning: decimation factor gives non-integer symbol length, sensitivity may be poor") + log.Println("Warning: decimated symbol length is non-integral, sensitivity may be poor") + } + + if d.DecCfg.SymbolLength < 3 { + log.Fatal("Error: illegal decimation factor, choose a smaller factor") } return