-
Notifications
You must be signed in to change notification settings - Fork 10
/
pton2mark
executable file
·168 lines (147 loc) · 6.86 KB
/
pton2mark
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
#!/bin/csh
#
# PTON2MARK (no inputs) OR
# PTON2MARK DDD YY, e.g. PTON2MARK 189 2019 (two inputs)
#
# Converts PTON weatherstation data to post it publicly on the Web.
# Converts SBF external-sensor binaries to ASCII and then to MRK, the
# format liked by Adam Wolf for intercomparison with the Arable Mark.
#
# This needs to be run on a machine where BIN2ASC is operational,
# unless your data have already been converted, in which case it will
# bypass BIN2SAC.
#
# See https://kb.unavco.org/kb/print-840.html
# Uses standard Unix tools and RxTools/timeconv
#
# Check the "Note" on L56 to make sure you're in the right place
#
# Last modified by fjsimons-at-alum.mit.edu, 09/21/2020
if($#argv == 1 || $#argv > 2)then
echo ' '
echo 'Zero or Two input arguments expected:'
echo ' '
echo 'If ZERO, works on the date of today. If TWO, then these:'
echo ' '
echo '----------> Julian day, e.g. 1 or 197 (for Jan 1 or July 16)'
echo '----------> Calendar year, e.g. 2019'
echo ' '
goto label999
endif
if($#argv == 0)then
# Make the Julian day and the current year, needs $PERL5LIB set so do not
# escape the environmental variable file loading by starting with csh -f
set DDD = `$home/PROGRAMS/LFILES/juliantoday.pl | awk '{printf "%3.3i", $5}'`
set YYYY = `date | awk '{print $6}'`
echo Running pton2mark $DDD $YYYY
else if($#argv == 2)then
# Take the Julian day and four-digit year from the input
set DDD = `printf "%3.3i" $1`
set YYYY = $2
endif
# Make the two-digit year
set YY = `echo $YYYY-2000 | bc`
# Find where the Raw/Ext SBF file is, depends on the host machine
if ($HOST == cressida.princeton.edu) then
set diro = /home/fjsimons/PTONcGPS
else if ($HOST == lemaitre) then
set diro = /u/fjsimons/CLASSES/FRS-Spain/SensorData/PTONcGPS
endif
# Set origin and destination folder
# Note that RawData might work also, both logs contain the weather data
set orig = $diro/RawData/$YYYY
set orig = $diro/ExtData
set dest = $diro/WeatherData/$YYYY
# Set input SBF file name
set fil1 = $orig/pton{$DDD}0.{$YY}_
# After file conversion you WILL get this whole new filename
set fil2 = $dest/pton{$DDD}0.{$YY}__SBF_ASCIIIn.txt
# Skip ahead if you do not have the input SBF file
test ! -f $fil1 && echo You do not have $fil1
test ! -f $fil1 && goto label888
# You might have made it before... don't redo it if you have
test ! -f $fil2 && echo Creating $fil2
# Now convert the file from SBF binary to ASCII
test ! -f $fil2 \
&& /opt/Septentrio/RxTools/bin/bin2asc -t -m ASCIIIn -f $fil1 -p $dest
# Flow control
label888:
# Skip ahead if you (still) do not have the output ASCII file
test ! -f $fil2 && echo You do not have $fil2
test ! -f $fil2 && goto label999
# These are the header variables that you will record
set headerline = 'Timestamp MeanWindDirection_deg MeanWindSpeed_mps AirTemp_C RelHum AirPress_bar RainAcc_mm HailAcc_hits'
# Process the weather file name, change SBF to ASC and make new extension MRK
set newfile = `echo $fil2 | sed 's/SBF/ASC/g'`
set newfile = $newfile:r.mrk
# Skip ahead if you already do have the MRK file
test -f $newfile && echo You do have $newfile
test -f $newfile && goto label998
set tempfl1 = $newfile:r.tmp1
set tempfl2 = $newfile:r.tmp2
echo Converting $fil2 to $newfile
# Convert the time stamp by extracting the first two columns... convert to milliseconds by nixing the period... write to tempfile
awk 'NR>2 {print}' $fil2 | sed 's/\.//g' | sed 's/[,=]/ /g' \
| awk '{printf "timeconv -t %s -w %s | grep UTC | awk '\''{print $5\"Z\"}'\'' | sed '\''s/-/T/g'\'' | sed '\''s/\\//-/g'\''\n",$1,$2}' \
>! $tempfl1
# Make temp file executable and then execute it, which makes another timefile
/bin/chmod +x $tempfl1
$tempfl1 >! $tempfl2
# Work on the remaining columns, which contains the weather string, which we strip of units and so on... write to tempfile
# The hashes are weird - possibly a write error, hopefully not a change of unit? Anyay, we get rid of it also.
# There is a failure mode where the command is not 65, e.g. day 164 2020 which so far I fixed by hand
awk 'NR>2 {print}' $fil2 \
| sed 's/Sync\/address error/Dm=NaND,Sm=NaNM,Ta=NaNC,Ua=NaNP,Pa=NaNB,Rc=NaNM,Hc=NaNM/g' \
| sed 's/Unknown cmd error/Dm=NaND,Sm=NaNM,Ta=NaNC,Ua=NaNP,Pa=NaNB,Rc=NaNM,Hc=NaNM/g' \
| sed 's/[,=]/ /g' \
| sed 's/[C,M,B,\#]//g' \
| sed 's/\BP//g' \
| sed 's/\BD//g' \
| awk '{printf "%s %s %s %s %s %s %s\n",$7,$9,$11,$13,$15,$17,$19}' >! $tempfl1
# Make a final file by concatenating a header line with the time stamp and the weather string from the two temp files
echo $headerline >! $newfile
paste $tempfl2 $tempfl1 | awk '{printf "%s %s %4s %5s %4s %6s %4s %4s\n",$1,$2,$3,$4,$5,$6,$7,$8}' >> $newfile
/bin/rm $tempfl2 $tempfl1
# Flow control
label998:
# Make the PDF plot
# Put this one file on the Web
# scp $newfile [email protected]:/home/www/people/simons/PTON
# Rather see to it that those get pushed onto the server via a
# passwordless single-command-use SSH key for back-up to a directory
# that is specified via the remote public key by prepending the
# relevant information in the authorized_keys file (for us here it is
# /home/www/people/simons/PTON on geoweb.princeton.edu), so that the
# pushing can be done via CRON without passwords being prompted
# Note that there better not any other key preceding with overriding preference,
# see ssh-add -l and the authorized_keys on the remote host... in which case the
# file goes to the home directory. If you have the correct key and you use rrsync
# the line below only specifies the directory in the remote authorized_keys...
# Maybe make sure to do
# ssh-add -d $home/.ssh/id_rsa_geoweb
# or else specify IdentitiesOnly in .ssh/config but for now, we just leave it.
# Note that in figuring this out, I did split the keys on cressida and lemaitre -
# rrsync only does take ONE command but it could take two comma-separated clients...
# which is how mermaid was setup... never mind, it's more epxlicit this way
set finaldest = /home/www/people/simons/PTON
rsync -ve "ssh -i $home/.ssh/id_remote_backup2" $newfile [email protected]:$finaldest
# Flow control
label999:
############## File Format Documentation ##########################################
# Every file begins with the two header lines,
# TOW [s],WNc [w],CD,StringLn,ASCIIString
# ---------------------------------------
# and then the data themselves, i.e.
# TOW [s] Time of Week, seconds
# WNc [w] GPS Week number
# COM port, a number, the 0R0 command code, and then met pack weather string:
# Dm=076D,Sm=0.1M,Ta=24.4C,Ua=20.9P,Pa=0.8429B,Rc=0.00M,Hc=0.0M
#
# Dm - wind direction, degrees
# Sm - wind speed, m/s
# Ta - temperature, degree C
# Ua - humidity, percent
# Pa - barometric pressure, Bar
# Rc - rainfall, mm/sampling period (reset at each measurement)
# Hc - hail, hits/sampling period (reset at each measurement)
########################################## ##########################################