diff --git a/docs/source/processing-levels.md b/docs/source/processing-levels.md index 37d566c77..cecd774d2 100644 --- a/docs/source/processing-levels.md +++ b/docs/source/processing-levels.md @@ -1,63 +1,7 @@ -# Proposed Echosounder Data Processing Levels (DRAFT) +# Echosounder data processing levels -The decades-long experience from the satellite remote sensing community has shown that a set of robust and well-articulated definitions of "data processing levels" [1]–[5] can lead directly to broad and highly productive use of data. Processing level designations also provide important context for data interpretation [6]. However, no such community agreement exists for active acoustic data. The ambiguity associated with the interoperability and inter-comparability of processed sonar data products has hindered efficient collaboration and integrative use of the rapidly growing data archive across research institutions and agencies. +The decades-long experience from the satellite remote sensing community has shown that a set of robust and well-articulated definitions of "data processing levels" {cite}`weaver_2014` can lead directly to broad and highly productive use of data. Processing level designations also provide important context for data interpretation {cite}`hills_2015`. However, no such community agreement exists for echosounder data. Improving the interoperability and inter-comparability of processed echosounder data products will likely facilitate efficient collaboration and integrative use of the rapidly growing data archive across research institutions and agencies. -The `echopype` team is developing a clearly defined progression of data processing levels for active ocean sonar data. The development leverages the collective experience from remote sensing and large-scale, long-term ocean and ecological observatories [7]–[10]. Data processing functions in `echopype` are clearly associated with specific Processing Level inputs and outputs, and when appropriate, will generate a `processing_level` dataset global attribute with entries such as "Level 1A", "Level 2B", "Level 4", etc. +To help address this gap, the echopype team is developing a clearly defined set of progression of echosounder data processing levels by leveraging the collective experience from remote sensing and large-scale, long-term ocean and ecological observatories. - -## Processing Levels and Sub-levels - -### Level 0 (L0) - -**Description:** Raw data in vendor sensor format. - -- raw binary files. Associated metadata may be found in external files. - -### Level 1 (L1) - -**Description:** Raw data packaged with ancillary information and converted and standardized to an open convention and standard data formats. May be distributed in the following two forms: - - as sets of individual converted files as originally segmented into arbitrary time ranges during sensor file creation, or - - compiled into larger granules corresponding to logical deployment intervals. - -- **L1A**: Raw L0 data converted to a standardized, open format with geographic coordinates (latitude & longitude) included. Includes other ancillary information extracted from sensor-generated L0 data or other external sources. May include environmental information such as temperature, salinity and pressure. Use of the SONAR-netcDF4 version 1 convention is strongly recommended. -- **L1B**: L1A data with quality-control steps applied, such as time-coordinate corrections that enforce strictly increasing, non-duplicate timestamps. - -### Level 2 (L2) - -**Description:** Calibrated acoustic quantities at raw data resolution, with spatial coordinates included (latitude, longitude and depth) - -- **L2A**: Volume backscattering strength (`Sv`) with interpolated latitude, longitude and depth coordinates. May incorporate addition information, such as split beam angle -- **L2B**: `Sv` L2A data with noise removal or other data filtering applied, including seafloor bottom removal. - -### Level 3 (L3) - -**Description:** Calibrated acoustic quantities regridded or aggregated to a common grid across channels. May include noise removal or other filtering. - -- `Sv` resampled to a common, high-resolution grid across channels -- Mean Volume Backscattering Strength (`MVBS`) -- Masks applied to `Sv` based on different scattering source classification criteria and methods, such as frequency difference across two channels. -- **L3A**: The above variables computed on L2A data -- **L3B**: The above variables computed on L2B (filtered) data - -### Level 4 (L4) - -**Description:** Acoustically derived biological features, involving further processing of L3 data that may include data reduction or incorporation of external sources of data. - -- Nautical Area Backscattering Coefficients (`NASC`), potentially partitioned into inferred sources of scattering -- Summary statistics of echogram features (center_of_mass, dispersion, etc) -- Taxon or species-level data labels (classification). May originate from a variety of methods, including frequency difference thresholds. -- Estimated biomass, combined or partitioned into different taxonomic sources. - - -## References - -- [1] Parkinson, C. L., A. Ward, and M. D. King (eds.). 2006. Earth science reference handbook: A guide to NASA’s Earth Science Program and Earth Observing Satellite Missions. National Aeronautics and Space Administration. https://atrain.nasa.gov/publications/2006ReferenceHandbook.pdf -- [2] NASA. 2021. Data Processing Levels | Earthdata. Last viewed Mar. 24, 2023. https://earthdata.nasa.gov/collaborate/open-data-services-and-software/data-information-policy/data-levels/ -- [3] Ramapriyan, H. K., and P. J. T. Leonard. 2021. Data Product Development Guide (DPDG) for Data Producers version1.1. NASA Earth Science Data and Information System Standards Office, 21 October 2021. https://doi.org/10.5067/DOC/ESO/RFC-041VERSION1 -- [4] Robinson, I. 2006. Satellite Measurements for Operational Ocean Models, pp. 147-189. In: Chassignet, E.P. and Verron, J. (eds). Ocean Weather Forecasting: An Integrated View of Oceanography. Springer, New York, NY. https://doi.org/10.1007/1-4020-4028-8_6 -- [5] Weaver, R. 2014. Processing Levels, pp. 517-520. In: Njoku, E.G. (ed). Encyclopedia of Remote Sensing. Encyclopedia of Earth Sciences Series. Springer, New York, NY. https://doi.org/10.1007/978-0-387-36699-9_36 -- [6] Hills, D. J., R. R. Downs, R. Duerr, J. C. Goldstein, M. A. Parsons, and H. K. Ramapriyan. 2015. The importance of data set provenance for science. Eos, 96, Published on 4 December 2015. https://doi.org/10.1029/2015EO040557 -- [7] Heaney, K., B. Martin, J. Miksis-Olds, T. Moore, J. Warren, and M. Ainslie. 2020. ADEON data processing specification. Version 1.0. Technical report by Applied Ocean Sciences for Prime Contract No. M16PC00003, Apr. 2020. https://adeon.unh.edu/sites/default/files/user-uploads/ADEON_Data%20Processing_Specification_FINAL.pdf -- [8] IFREMER. 2019. Processing Levels - Oceanographic Data. Last viewed Mar. 24, 2023. http://en.data.ifremer.fr/All-about-data/Data-management/Processing-levels -- [9] NEON. 2023 Data Processing | NSF NEON | Open Data to Understand our Ecosystems. Last viewed Mar. 24, 2023. https://www.neonscience.org/data-samples/data-management/data-processing -- [10] OOI. 2023. Ocean Observatories Initiative: Glossary - Data Terminology. Ocean Observatories Initiative. Last viewed Mar. 24, 2023. https://oceanobservatories.org/glossary/#DATATERMS +Check out the [echolevels](http://echolevels.readthedocs.io/) document for the current status, and please chime in with any [input or questions via GitHub issues](https://github.com/uw-echospace/data-processing-levels/issues/new)! diff --git a/docs/source/references.bib b/docs/source/references.bib index 6541483e2..54d82d613 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -8,6 +8,7 @@ @article{DeRobertis2007_noise year = {2007}, pages = {1282--1291}, } + @techreport{Demer2015, title = {Calibration of acoustic instruments}, institution = {ICES Cooperative Research Report, No. 326}, @@ -16,6 +17,7 @@ @techreport{Demer2015 doi = {10.17895/ices.pub.5494}, pages = {133 pp}, } + @article{MacKenzie1981, title = {Nine-term equation for sound speed in the oceans}, volume = {70}, @@ -26,6 +28,7 @@ @article{MacKenzie1981 pages = {807--812}, doi = {10.1121/1.386920}, } + @article{Ainslie1998, title = {A simplified formula for viscous and chemical absorption in sea water}, volume = {103}, @@ -39,3 +42,31 @@ @article{Ainslie1998 note = {Publisher: Acoustical Society of America}, pages = {1671--1672}, } + +@incollection{weaver_2014, + address = {New York, NY}, + series = {Encyclopedia of {Earth} {Sciences} {Series}}, + title = {Processing {Levels}}, + isbn = {978-0-387-36699-9}, + language = {en}, + booktitle = {Encyclopedia of {Remote} {Sensing}}, + publisher = {Springer}, + author = {Weaver, Ron}, + editor = {Njoku, Eni G.}, + year = {2014}, + doi = {10.1007/978-0-387-36699-9_36}, + pages = {517--520}, + file = {Full Text PDF:/Users/wujung/Zotero/storage/K8CUALCF/Weaver - 2014 - Processing Levels.pdf:application/pdf}, +} + +@misc{hills_2015, + title = {The {Importance} of {Data} {Set} {Provenance} for {Science}}, + url = {http://eos.org/opinions/the-importance-of-data-set-provenance-for-science}, + abstract = {Data do not exist in a vacuum. To be useful, data must be accompanied by context on how they are captured, processed, analyzed, and validated and other information that enables interpretation and use.}, + language = {en-US}, + urldate = {2023-10-23}, + journal = {Eos}, + author = {Hills, D. J. and Downs, R. R. and Duerr, R. and Goldstein, J. C. and Parsons, M. A. and Ramapriyan, H. K.}, + month = dec, + year = {2015}, +}