Skip to content

Commit

Permalink
create and use an index db from ogimet
Browse files Browse the repository at this point in the history
  • Loading branch information
@flyingeek
flyingeek committed Mar 12, 2024
1 parent dab1acc commit f08f601
Show file tree
Hide file tree
Showing 7 changed files with 125 additions and 60 deletions.
3 changes: 2 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@flyingeek/lidojs",
"version": "1.6.66",
"version": "1.6.67",
"description": "convert Lido OFP text files",
"publishConfig": {
"registry": "https://npm.pkg.github.com/"
Expand All @@ -21,6 +21,7 @@
"build_with_wmo": "webpack && node src/makeWmo.js",
"makefish": "node src/makeFish.js",
"makewmo": "node src/makeWmo.js",
"updateogimet": "node src/updateOgimetIdx.js",
"release": "npm version patch && webpack"
},
"browserslist": [
Expand Down
120 changes: 65 additions & 55 deletions src/makeWmo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const Papa = require('papaparse');
const fs = require('fs');
const path = require('path');
const {https} = require('follow-redirects');
const ogimetIds = require('./ogimet_idx.json');

const wmoPath = "./dist/wmo.json";
const wmoVarPath = "./dist/wmo.var.js";
Expand All @@ -14,8 +15,12 @@ const volaURL = "https://gist.github.com/flyingeek/54caad59410a1f4641d480473ec82
//const volaURL = "https://oscar.wmo.int/oscar/vola/vola_legacy_report.txt";
const volaJSONURL = "https://gist.githubusercontent.com/flyingeek/54caad59410a1f4641d480473ec824c3/raw/oscar_wmo_stations.json"
//const volaJSONURL = "https://oscar.wmo.int/surface/rest/api/search/station?facilityType=landFixed&programAffiliation=GOSGeneral,RBON,GBON,RBSN,RBSNp,RBSNs,RBSNsp,RBSNst,RBSNt,ANTON,ANTONt&variable=216&variable=224&variable=227&variable=256&variable=310&variable=12000";
const excludedStations = ['71822', '41298', '72232', '41284', '71944', '41248', '41274', '71872', '83032', '83075', '83249', '83581', '83742', '40280', 'LFPC', '04418', '81003', 'LIMT', '84378', '04282', '11336'];
//['71822', '72232', '71944', '83032', '83075', '83249', '83581', '83742', '40280', 'LFPC', '04418', '81003', 'LIMT', '84378', '04282', '11336']
const excludedStations = [];

console.log(`${excludedStations.length} excluded stations`);
console.log(`${ogimetIds.length} ogimet stations`);
console.log(`consider updating ogimet stations with npm run updateogimet`);

/**
* Promise to oscar json importer
*/
Expand All @@ -34,26 +39,26 @@ function volaJSONRequest(url) {
wmo = JSON.parse(data).stationSearchResults;
const results = {};
let counter = 0;
let knownByOgimetCounter = 0;
wmo.forEach(w => {
const wid = w.wigosId.split('-').pop();
if (
wid.match(/^\d{5}$/u)

/*
&& w.stationStatusCode === 'operational'
&& w.stationAssessedStatusCode === 'operational'
&& (w.stationProgramsDeclaredStatuses.indexOf('GOS')>=0)
*/
&& w.wigosId.startsWith('0-20000-0-')
) {
results[wid] =[
parseFloat(w.longitude),
parseFloat(w.latitude),
w.stationStatusCode
];
counter += 1;
if (ogimetIds.indexOf(wid) >= 0) {
knownByOgimetCounter += 1;
if (results[wid]) console.log(`doublon pour ${wid}`);
results[wid] =[
parseFloat(w.longitude),
parseFloat(w.latitude),
w.stationStatusCode
];
}
}
});
console.log(`${counter} oscar wmo stations`);
console.log(`${counter} oscar wmo stations / ${knownByOgimetCounter} known by ogimet`);
resolve(results);
});
});
Expand Down Expand Up @@ -85,20 +90,28 @@ function volaRequest(url) {
return sign * (degrees + (minutes / 60) + (seconds / 3600));
};
let counter = 0;
let knownByOgimetCounter = 0;
parsed.data.slice(1).forEach((row) => {
if (row.length < 28) return;
const wid = row[5];
if (!wid) return;
// wid: [lon, lat, remarks]
results[wid] = [
normalize(row[9]),
normalize(row[8]),
row[28]//.split(", ")
]
counter += 1;
//console.log({wid, "data": results[wid]});
if (wid && wid.match(/^\d{5}$/u) && row[6] === "0") {
if (results[wid]) {
if (wid !== "94907") console.log(`doublon pour ${wid}`); //known doubles
} else {
counter += 1;
if (ogimetIds.indexOf(wid) >= 0) {
knownByOgimetCounter += 1;
// wid: [lon, lat, remarks]
results[wid] = [
normalize(row[9]),
normalize(row[8]),
row[28]//.split(", ")
]
}
}
}
});
console.log(`${counter} vola stations`);
console.log(`${counter} vola stations / ${knownByOgimetCounter} known by ogimet`);
resolve(results);
})
});
Expand Down Expand Up @@ -130,18 +143,23 @@ function wmoRequest(url) {
return sign * (degrees + (minutes / 60) + (seconds / 3600));
};
let counter = 0;
let knownByOgimetCounter = 0;
parsed.data.forEach((row) => {
if (row.length < 8) return;
const wid = row[0] + row[1];
// wid, name, lon, lat
results.push([
row[0] + row[1],
(row[2] === '----') ? row[0] + row[1] : row[2],
normalize(row[8]),
normalize(row[7])
]);
counter += 1;
if (ogimetIds.indexOf(wid) >= 0) {
knownByOgimetCounter += 1;
results.push([
wid,
(row[2] === '----') ? row[0] + row[1] : row[2],
normalize(row[8]),
normalize(row[7])
]);
}
});
console.log(`${counter} wmo stations`);
console.log(`${counter} wmo stations / ${knownByOgimetCounter} known by ogimet`);
resolve(results);
})
});
Expand Down Expand Up @@ -171,37 +189,29 @@ async function mergeData() {
} else {
data[geohash] = [value];
}
counter += 1;
}

await Promise.all([volaRequest(volaURL), wmoRequest(wmoURL), volaJSONRequest(volaJSONURL)]).then(([volaData, wmoData, oscarData]) => {
const wmoIds = [];
const addedWmoIds = [];
for (const [wid, name, lon, lat] of wmoData) {
if (wid in volaData && (excludedStations.indexOf(wid) < 0) && (excludedStations.indexOf(name) < 0)){
const [volaLon, volaLat, remarks] = volaData[wid];
if (['CYMT', 'LFBV'].indexOf(name) >= 0) continue;
if (remarks.indexOf("GOS") < 0) continue;
if (oscarData[wid] && oscarData[wid][2] !== 'operational') {
// console.log(`${wid} is ${oscarData[wid][2]}, skipping`);
continue;
}
if (Math.abs(volaLon - lon) > 0.1 || Math.abs(volaLat - lat) > 0.1) {
continue;
}
wmoIds.push(wid);
if ((excludedStations.indexOf(wid) >= 0) || (excludedStations.indexOf(name) >= 0)) continue;
addedWmoIds.push(wid);
addData(name, lat, lon);
counter += 1;
}
}
for (const [wid, [lon, lat, remarks]] of Object.entries(volaData)) {
if ((wmoIds.indexOf(wid) < 0) && (excludedStations.indexOf(wid) < 0) && (remarks.indexOf("GOS") >= 0)) {
if (oscarData[wid] && oscarData[wid][2] !== 'operational') {
// console.log(`${wid} is ${oscarData[wid][2]}, skipping`);
continue;
}
addData(wid, lat, lon);
counter += 1;
}
console.log(`wmo stations processed, total: ${counter} WMO stations`);
for (const [wid, [lon, lat, ]] of Object.entries(volaData)) {
if (excludedStations.indexOf(wid) >= 0 || addedWmoIds.indexOf(wid) >= 0) continue;
addedWmoIds.push(wid);
addData(wid, lat, lon);
}
console.log(`vola stations processed, total: ${counter} WMO stations`);
for (const [wid, [lon, lat, ]] of Object.entries(oscarData)) {
if (excludedStations.indexOf(wid) >= 0 || addedWmoIds.indexOf(wid) >= 0) continue;
addedWmoIds.push(wid);
addData(wid, lat, lon);
}
console.log(`oscar stations processed, total: ${counter} WMO stations`);
})
return data;
}
Expand Down
1 change: 1 addition & 0 deletions src/ogimet_idx.json
View file

Large diffs are not rendered by default.

53 changes: 53 additions & 0 deletions src/updateOgimetIdx.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
const fs = require("fs");
const {https} = require('follow-redirects');
const Papa = require('papaparse');

/* CHECK TESTS AFTER UPDATING */
const outputPath = "./src/ogimet_idx.json";

/*
returns date in the format YYYYMMDDHH00, the hours parameters indicates the offset in hour from now
*/
const beginParameter = (hours) => {
const ts = Date.now() + (hours * 3600000);
return (new Date(ts)).toISOString()
.replace(/[^\d]+/gu,'')
.slice(0,10) + "00";
}
const ogimetUrl = `https://www.ogimet.com/cgi-bin/getsynop?begin=${beginParameter(-12)}&lang=eng&header=yes`;

/**
Process Ogimet SYNOP request and extract SYNOP IDs
*/
function ogimetRequest(url) {
return new Promise((resolve, reject) => {
https.get(url, (response) => {
let data = "";
const stations = new Set();
response.on("data", (chunk) => {
data += chunk;
});
response.on("error", (err) => {
reject(err);
});
response.on("end", () => {
const parsed = Papa.parse(data);
parsed.data.slice(1).forEach((row) => {
if (row[0].match(/^\d{5}$/u)) {
stations.add(row[0]);
}
});
resolve(stations);
});
});
});
}
ogimetRequest(ogimetUrl).then(stations => {
fs.writeFile(outputPath, JSON.stringify([...stations]), (err) => {
if (err) {
throw err;
} else {
console.log(`Saved ${stations.size} stations!`);
}
});
});
2 changes: 1 addition & 1 deletion testwmo/ofp_AF010_27Sep2019_nvp.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ test('ogimetRoute', () => {
.filter(p => p.name !== "")
.map(p => p.name)
.join(' ');
expect(names).toEqual("LFPG LFPB 07002 03559 03354 EGNH 03916 03980 71513 71634 KNHZ KPVD 72501 72505 KJFK");
expect(names).toEqual("LFPG LFPB 07002 03680 03560 03354 EGNH 03916 CWXW CYNA CWRZ KPWM KBOS KPVD KJFK");
});
4 changes: 2 additions & 2 deletions testwmo/ofp_AF191_30Dec2016.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ test('ogimetRoute', () => {
.filter(p => p.name !== "")
.map(p => p.name)
.join(' ');
expect(names).toEqual("43296 43264 43161 43158 OIKB 40851 40821 OIIK OITZ OITT 17024 15561 15499 15460 15182 11880 11628 11406 10605 06484 LFPG");
expect(names).toEqual("43296 43272 VOBI 43161 43158 OIKB 40851 OIYY OIII OITZ OITT LTCE 17024 15460 15230 11880 LKTB 11406 EDDF EDFH LFPG");
});

test('ogimetData', () => {
const data = ogimetData(ofp, wmoGrid);
expect(data.wmo.join(' ')).toEqual("43296 43264 43161 43158 OIKB 40851 40821 OIIK OITZ OITT 17024 15561 15499 15460 15182 11880 11628 11406 10605 06484 LFPG");
expect(data.wmo.join(' ')).toEqual("43296 43272 VOBI 43161 43158 OIKB 40851 OIYY OIII OITZ OITT LTCE 17024 15460 15230 11880 LKTB 11406 EDDF EDFH LFPG");
expect(data.url.length).toBeGreaterThan(10);
});
2 changes: 1 addition & 1 deletion testwmo/ogimet.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ test('getNearestPointsDirty', () => {
}).toThrow();
wmoGrid.data = loadWmo();
nearest = Array.from(wmoGrid.getNearestPointsDirty(centerPoint, 75, km_to_rad));
expect(nearest.length).toEqual(41);
expect(nearest.length).toEqual(40);
});

test('getNearestPoints', () => {
Expand Down

0 comments on commit f08f601

Please sign in to comment.