From 875fb3ccefc5a6c586b62a5c63ff7dad524ca4be Mon Sep 17 00:00:00 2001 From: jianjunj Date: Thu, 16 Jan 2025 12:55:38 -0500 Subject: [PATCH 1/3] Add precipitable clouds in microwave all-sky assimilations (#63) Add rain, snow, and graupel clouds for CRTM inputs. Add a prior filter to zero clouds over non-water surfaces. Add an option of clouds in the Hydrometeo check. Add gross error check threshold for precipition impacted channels. Add cold-air-outbreak filters. Read cloud fraction from GeoVals. Render list of not-bias corrected channels from JCB tables. Add configurations for atms_n21. --- .../atmosphere/atms_n21.yaml | 43 ++ observations/atmosphere/amsua_n19.yaml.j2 | 142 +++- observations/atmosphere/atms_n20.yaml.j2 | 157 +++- observations/atmosphere/atms_n21.yaml.j2 | 686 ++++++++++++++++++ observations/atmosphere/atms_npp.yaml.j2 | 141 +++- 5 files changed, 1155 insertions(+), 14 deletions(-) create mode 100644 observation_chronicle/atmosphere/atms_n21.yaml create mode 100644 observations/atmosphere/atms_n21.yaml.j2 diff --git a/observation_chronicle/atmosphere/atms_n21.yaml b/observation_chronicle/atmosphere/atms_n21.yaml new file mode 100644 index 0000000..3311e65 --- /dev/null +++ b/observation_chronicle/atmosphere/atms_n21.yaml @@ -0,0 +1,43 @@ +# Instrument metadata +# ------------------- +commissioned: 2023-05-01T12:00:00 + +observer_type: satellite # Type of chronicle to use + +# Instrument initial configuration +# -------------------------------- +channel_variables: + simulated: min + active: min + biascorrtd: min + error: max + error_cld: max + ermax: max + var_b: min + var_pg: min + icld_det: min + icloud: min + iaerosol: min +channel_values: + 1: [ 1, 1, 1, 4.50, 20.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 2: [ 1, 1, 1, 4.50, 25.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 3: [ 1, 1, 1, 4.50, 12.00, 3.00, 10.00, 0.00, -2, 1, -1 ] + 4: [ 1, 1, 1, 2.50, 7.00, 3.00, 10.00, 0.00, -2, 1, -1 ] + 5: [ 1, 1, 1, 0.55, 3.50, 1.00, 10.00, 0.00, -2, 1, -1 ] + 6: [ 1, 1, 1, 0.30, 3.00, 1.00, 10.00, 0.00, -2, 1, -1 ] + 7: [ 1, 1, 1, 0.30, 0.80, 1.00, 10.00, 0.00, -2, 1, -1 ] + 8: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 9: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 10: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 11: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] + 12: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] + 13: [ 1, 1, 1, 0.55, 0.55, 1.00, 10.00, 0.00, -2, 1, -1 ] + 14: [ 1, 1, 1, 0.80, 0.80, 2.00, 10.00, 0.00, -2, 1, -1 ] + 15: [ 1, 1, 0, 4.00, 4.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 16: [ 1, 1, 1, 4.00, 19.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 17: [ 1, 1, 1, 4.00, 30.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 18: [ 1, 1, 1, 3.50, 25.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 19: [ 1, 1, 1, 3.00, 16.50, 2.00, 10.00, 0.00, -2, 1, -1 ] + 20: [ 1, 1, 1, 3.00, 12.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 21: [ 1, 1, 1, 3.00, 9.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 22: [ 1, 1, 1, 3.00, 6.50, 2.00, 10.00, 0.00, -2, 1, -1 ] diff --git a/observations/atmosphere/amsua_n19.yaml.j2 b/observations/atmosphere/amsua_n19.yaml.j2 index dba4162..2cc30e3 100644 --- a/observations/atmosphere/amsua_n19.yaml.j2 +++ b/observations/atmosphere/amsua_n19.yaml.j2 @@ -21,9 +21,10 @@ # -------------------- obs operator: name: CRTM - Absorbers: [H2O, O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Absorbers: [H2O, O3, CO2] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id amsua_n19 EndianType: little_endian @@ -34,6 +35,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -64,7 +67,25 @@ # Observation Filters (QC) # ------------------------ - obs filters: + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + + obs post filters: - filter: BlackList filter variables: - name: brightnessTemperature @@ -124,6 +145,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *{{observation_from_jcb}}_clouds obserr_clearsky: *{{observation_from_jcb}}_err0 clwret_function: name: ObsFunction/CLWRetMW @@ -301,6 +323,8 @@ x1: *{{observation_from_jcb}}_x1 err0: *{{observation_from_jcb}}_err0 err1: *{{observation_from_jcb}}_err1 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [{{ get_satellite_variable(observation_from_jcb, "ermax") }}] action: name: reject @@ -334,8 +358,116 @@ minvalue: 1.0e-12 action: name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 - +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n20.yaml.j2 b/observations/atmosphere/atms_n20.yaml.j2 index c16a22c..82d2f50 100644 --- a/observations/atmosphere/atms_n20.yaml.j2 +++ b/observations/atmosphere/atms_n20.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n20 @@ -37,6 +37,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -65,6 +67,25 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -114,6 +135,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [ObsValue] + # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -125,6 +147,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [HofX] + # Calculate symmetric retrieved CLW - filter: Variable Assignment assignments: @@ -166,6 +189,7 @@ scatret_ch314: 2 scatret_ch890: 16 scatret_types: [ObsValue] + # Calculate CLW obs/bkg match index - filter: Variable Assignment assignments: @@ -184,6 +208,7 @@ clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, 0.050, 0.100] + # Calculate symmetric observation error - filter: Variable Assignment assignments: @@ -209,6 +234,7 @@ err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] + # Calculate Innovation@DerivedMetaData - filter: Variable Assignment assignments: @@ -225,6 +251,7 @@ - name: brightnessTemperature@HofX channels: *{{observation_from_jcb}}_simulated_channels coefs: [1, -1] + # Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: @@ -235,6 +262,7 @@ error function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: @@ -322,6 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -348,6 +377,7 @@ options: sensor: *{{observation_from_jcb}}_sensor_id channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -357,6 +387,7 @@ inflation variable: name: ObsErrorFactorTopo@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: @@ -368,6 +399,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -377,6 +409,7 @@ inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: @@ -391,6 +424,7 @@ channels: *{{observation_from_jcb}}_simulated_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -400,6 +434,7 @@ inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 9: Situation Dependent Check - filter: Variable Assignment assignments: @@ -427,6 +462,7 @@ obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -436,8 +472,9 @@ inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threhold + # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -446,6 +483,7 @@ name: ObsErrorFactorLatRad@ObsFunction options: latitude_parameters: [25.0, 0.25, 0.04, 3.0] + - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData @@ -470,9 +508,11 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threhold: 3 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] + - filter: Background Check filter variables: - name: brightnessTemperature @@ -526,7 +566,116 @@ ignore: rejected observations - name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n21.yaml.j2 b/observations/atmosphere/atms_n21.yaml.j2 new file mode 100644 index 0000000..560be3b --- /dev/null +++ b/observations/atmosphere/atms_n21.yaml.j2 @@ -0,0 +1,686 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: ATMS N21 + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &{{observation_from_jcb}}_simulated_channels {{ get_satellite_variable(observation_from_jcb, "simulated") }} + + # Observation Operator + # -------------------- + obs operator: + name: CRTM + Absorbers: [H2O, O3, CO2] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n21 + EndianType: little_endian + CoefficientPath: "{{crtm_coefficient_path}}" + linear obs operator: + Absorbers: [H2O, O3] + + # Observation Bias Correction (VarBC) + # ----------------------------------- + obs bias: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &{{observation_from_jcb}}_tlapse "{{atmosphere_obsbiasin_path}}/{{atmosphere_obstlapsein_prefix}}{{observation_from_jcb}}{{atmosphere_obstlapsein_suffix}}" + - name: lapseRate + tlapse: *{{observation_from_jcb}}_tlapse + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiascovin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovin_suffix}}" + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + + # Observation Pre Filters (QC) + # ---------------------------- + obs pre filters: + # Step 0-A: Create Diagnostic Flags + - filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + + # Observation Post Filters (QC) + # ----------------------------- + obs post filters: + # Step 0-B: Calculate derived variables + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + + # Calculate symmetric retrieved CLW + - filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + + - filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + + # Calculate scattering index from observation + - filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + + # Calculate CLW obs/bkg match index + - filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, + 0.030, 0.030, 0.050, 0.100] + + # Calculate symmetric observation error + - filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [0.350, 0.380, 0.400, 0.450, 0.500, 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, + 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, + 12.000, 9.000, 6.500] + + # Calculate Innovation@DerivedMetaData + - filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *{{observation_from_jcb}}_simulated_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *{{observation_from_jcb}}_simulated_channels + - name: brightnessTemperature@HofX + channels: *{{observation_from_jcb}}_simulated_channels + coefs: [1, -1] + + # Step 0-C: Assign Initial All-Sky Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 1: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + + # Step 2: Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject + + # Step 3A: CLW Retrieval Check (observation_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 3B: CLW Retrieval Check (background_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 4: Window Channel Sanity Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + + # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, + 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, + 3.000, 3.000, 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *{{observation_from_jcb}}_clouds + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *{{observation_from_jcb}}_simulated_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + + # Step 6: Observation Error Inflation based on Topography Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 7: Obs Error Inflation based on TOA Transmittancec Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 8: Observation Error Inflation based on Surface Jacobian Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 9: Situation Dependent Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, + 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, + 3.000, 3.000, 3.000, 3.000] + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + + # Step 10: Gross check + # Remove data if abs(Obs-HofX) > absolute threshold + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + + - filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + threshold: 3 + threshold_precip: 2.5 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] + + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + + # Step 11: Inter-Channel Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use passive_bc: true + sensor: *{{observation_from_jcb}}_sensor_id + use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, + 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + + # Step 12: Useflag Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, + 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 + +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject + # GeoVaLs for Driving Observation Operators (testing mode) + # -------------------------------------------------------- + geovals: + filename: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}_geoval{{atmosphere_obsdatain_suffix}}" + + # Passed benchmark for UFO testing + # -------------------------------- + passedBenchmark: 0 diff --git a/observations/atmosphere/atms_npp.yaml.j2 b/observations/atmosphere/atms_npp.yaml.j2 index d18985c..309da34 100644 --- a/observations/atmosphere/atms_npp.yaml.j2 +++ b/observations/atmosphere/atms_npp.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_npp @@ -37,6 +37,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -65,6 +67,25 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -329,7 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - + Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -453,7 +474,7 @@ channels: *{{observation_from_jcb}}_simulated_channels # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threhold + # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -487,7 +508,8 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threhold: 3 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] @@ -544,7 +566,116 @@ ignore: rejected observations - name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: From 0150d282785e3b48d62d7751bc906439baf88125 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Fri, 17 Jan 2025 21:41:42 -0500 Subject: [PATCH 2/3] Revert " Add precipitable clouds in microwave all-sky assimilations" (#68) --- .../atmosphere/atms_n21.yaml | 43 -- observations/atmosphere/amsua_n19.yaml.j2 | 142 +--- observations/atmosphere/atms_n20.yaml.j2 | 157 +--- observations/atmosphere/atms_n21.yaml.j2 | 686 ------------------ observations/atmosphere/atms_npp.yaml.j2 | 141 +--- 5 files changed, 14 insertions(+), 1155 deletions(-) delete mode 100644 observation_chronicle/atmosphere/atms_n21.yaml delete mode 100644 observations/atmosphere/atms_n21.yaml.j2 diff --git a/observation_chronicle/atmosphere/atms_n21.yaml b/observation_chronicle/atmosphere/atms_n21.yaml deleted file mode 100644 index 3311e65..0000000 --- a/observation_chronicle/atmosphere/atms_n21.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Instrument metadata -# ------------------- -commissioned: 2023-05-01T12:00:00 - -observer_type: satellite # Type of chronicle to use - -# Instrument initial configuration -# -------------------------------- -channel_variables: - simulated: min - active: min - biascorrtd: min - error: max - error_cld: max - ermax: max - var_b: min - var_pg: min - icld_det: min - icloud: min - iaerosol: min -channel_values: - 1: [ 1, 1, 1, 4.50, 20.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 2: [ 1, 1, 1, 4.50, 25.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 3: [ 1, 1, 1, 4.50, 12.00, 3.00, 10.00, 0.00, -2, 1, -1 ] - 4: [ 1, 1, 1, 2.50, 7.00, 3.00, 10.00, 0.00, -2, 1, -1 ] - 5: [ 1, 1, 1, 0.55, 3.50, 1.00, 10.00, 0.00, -2, 1, -1 ] - 6: [ 1, 1, 1, 0.30, 3.00, 1.00, 10.00, 0.00, -2, 1, -1 ] - 7: [ 1, 1, 1, 0.30, 0.80, 1.00, 10.00, 0.00, -2, 1, -1 ] - 8: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] - 9: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] - 10: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] - 11: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] - 12: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] - 13: [ 1, 1, 1, 0.55, 0.55, 1.00, 10.00, 0.00, -2, 1, -1 ] - 14: [ 1, 1, 1, 0.80, 0.80, 2.00, 10.00, 0.00, -2, 1, -1 ] - 15: [ 1, 1, 0, 4.00, 4.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 16: [ 1, 1, 1, 4.00, 19.00, 4.50, 10.00, 0.00, -2, 1, -1 ] - 17: [ 1, 1, 1, 4.00, 30.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 18: [ 1, 1, 1, 3.50, 25.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 19: [ 1, 1, 1, 3.00, 16.50, 2.00, 10.00, 0.00, -2, 1, -1 ] - 20: [ 1, 1, 1, 3.00, 12.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 21: [ 1, 1, 1, 3.00, 9.00, 2.00, 10.00, 0.00, -2, 1, -1 ] - 22: [ 1, 1, 1, 3.00, 6.50, 2.00, 10.00, 0.00, -2, 1, -1 ] diff --git a/observations/atmosphere/amsua_n19.yaml.j2 b/observations/atmosphere/amsua_n19.yaml.j2 index 2cc30e3..dba4162 100644 --- a/observations/atmosphere/amsua_n19.yaml.j2 +++ b/observations/atmosphere/amsua_n19.yaml.j2 @@ -21,10 +21,9 @@ # -------------------- obs operator: name: CRTM - Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 - Cloud_Seeding: true + Absorbers: [H2O, O3] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id amsua_n19 EndianType: little_endian @@ -35,8 +34,6 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -67,25 +64,7 @@ # Observation Filters (QC) # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - - obs post filters: + obs filters: - filter: BlackList filter variables: - name: brightnessTemperature @@ -145,7 +124,6 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds obserr_clearsky: *{{observation_from_jcb}}_err0 clwret_function: name: ObsFunction/CLWRetMW @@ -323,8 +301,6 @@ x1: *{{observation_from_jcb}}_x1 err0: *{{observation_from_jcb}}_err0 err1: *{{observation_from_jcb}}_err1 - threshold: 3 - threshold_precip: 2.5 obserr_bound_max: [{{ get_satellite_variable(observation_from_jcb, "ermax") }}] action: name: reject @@ -358,116 +334,8 @@ minvalue: 1.0e-12 action: name: reject -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-6, 15 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject + # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n20.yaml.j2 b/observations/atmosphere/atms_n20.yaml.j2 index 82d2f50..c16a22c 100644 --- a/observations/atmosphere/atms_n20.yaml.j2 +++ b/observations/atmosphere/atms_n20.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n20 @@ -37,8 +37,6 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -67,25 +65,6 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" - # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -135,7 +114,6 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [ObsValue] - # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -147,7 +125,6 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [HofX] - # Calculate symmetric retrieved CLW - filter: Variable Assignment assignments: @@ -189,7 +166,6 @@ scatret_ch314: 2 scatret_ch890: 16 scatret_types: [ObsValue] - # Calculate CLW obs/bkg match index - filter: Variable Assignment assignments: @@ -208,7 +184,6 @@ clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, 0.050, 0.100] - # Calculate symmetric observation error - filter: Variable Assignment assignments: @@ -234,7 +209,6 @@ err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] - # Calculate Innovation@DerivedMetaData - filter: Variable Assignment assignments: @@ -251,7 +225,6 @@ - name: brightnessTemperature@HofX channels: *{{observation_from_jcb}}_simulated_channels coefs: [1, -1] - # Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: @@ -262,7 +235,6 @@ error function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: @@ -350,7 +322,6 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -377,7 +348,6 @@ options: sensor: *{{observation_from_jcb}}_sensor_id channels: *{{observation_from_jcb}}_simulated_channels - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -387,7 +357,6 @@ inflation variable: name: ObsErrorFactorTopo@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: @@ -399,7 +368,6 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -409,7 +377,6 @@ inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: @@ -424,7 +391,6 @@ channels: *{{observation_from_jcb}}_simulated_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -434,7 +400,6 @@ inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 9: Situation Dependent Check - filter: Variable Assignment assignments: @@ -462,7 +427,6 @@ obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] - - filter: Perform Action filter variables: - name: brightnessTemperature @@ -472,9 +436,8 @@ inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threshold + # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -483,7 +446,6 @@ name: ObsErrorFactorLatRad@ObsFunction options: latitude_parameters: [25.0, 0.25, 0.04, 3.0] - - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData @@ -508,11 +470,9 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threshold: 3 - threshold_precip: 2.5 + threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - filter: Background Check filter variables: - name: brightnessTemperature @@ -566,116 +526,7 @@ ignore: rejected observations - name: reject -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n21.yaml.j2 b/observations/atmosphere/atms_n21.yaml.j2 deleted file mode 100644 index 560be3b..0000000 --- a/observations/atmosphere/atms_n21.yaml.j2 +++ /dev/null @@ -1,686 +0,0 @@ -- - - # Observation Space (I/O) - # ----------------------- - obs space: - name: ATMS N21 - obsdatain: - engine: - type: H5File - obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" - obsdataout: - engine: - type: H5File - obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" - io pool: - max pool size: 1 - simulated variables: [brightnessTemperature] - channels: &{{observation_from_jcb}}_simulated_channels {{ get_satellite_variable(observation_from_jcb, "simulated") }} - - # Observation Operator - # -------------------- - obs operator: - name: CRTM - Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 - Cloud_Seeding: true - obs options: - Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n21 - EndianType: little_endian - CoefficientPath: "{{crtm_coefficient_path}}" - linear obs operator: - Absorbers: [H2O, O3] - - # Observation Bias Correction (VarBC) - # ----------------------------------- - obs bias: - input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" - output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} - variational bc: - predictors: - - name: constant - - name: lapseRate - order: 2 - tlapse: &{{observation_from_jcb}}_tlapse "{{atmosphere_obsbiasin_path}}/{{atmosphere_obstlapsein_prefix}}{{observation_from_jcb}}{{atmosphere_obstlapsein_suffix}}" - - name: lapseRate - tlapse: *{{observation_from_jcb}}_tlapse - - name: emissivityJacobian - - name: sensorScanAngle - order: 4 - - name: sensorScanAngle - order: 3 - - name: sensorScanAngle - order: 2 - - name: sensorScanAngle - covariance: - minimal required obs number: 20 - variance range: [1.0e-6, 10.0] - step size: 1.0e-4 - largest analysis variance: 10000.0 - prior: - input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiascovin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovin_suffix}}" - inflation: - ratio: 1.1 - ratio for small dataset: 2.0 - output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" - - # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - - # Observation Pre Filters (QC) - # ---------------------------- - obs pre filters: - # Step 0-A: Create Diagnostic Flags - - filter: Create Diagnostic Flags - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - flags: - - name: ScanEdgeRemoval - initial value: false - force reinitialization: false - - name: Thinning - initial value: false - force reinitialization: false - - name: CLWRetrievalCheck - initial value: false - force reinitialization: false - - name: WindowChannelExtremeResidual - initial value: false - force reinitialization: false - - name: HydrometeorCheck - initial value: false - force reinitialization: false - - name: GrossCheck - initial value: false - force reinitialization: false - - name: InterChannelConsistency - initial value: false - force reinitialization: false - - name: UseflagCheck - initial value: false - force reinitialization: false - - # Observation Post Filters (QC) - # ----------------------------- - obs post filters: - # Step 0-B: Calculate derived variables - # Calculate CLW retrieved from observation - - filter: Variable Assignment - assignments: - - name: CLWRetFromObs@DerivedMetaData - type: float - function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [ObsValue] - - # Calculate CLW retrieved from observation - - filter: Variable Assignment - assignments: - - name: CLWRetFromBkg@DerivedMetaData - type: float - function: - name: CLWRetMW@ObsFunction - options: - clwret_ch238: 1 - clwret_ch314: 2 - clwret_types: [HofX] - - # Calculate symmetric retrieved CLW - - filter: Variable Assignment - assignments: - - name: CLWRetSymmetric@DerivedMetaData - type: float - value: 1000.0 - - - filter: Variable Assignment - where: - - variable: - name: CLWRetFromObs@DerivedMetaData - minvalue: 0. - maxvalue: 999. - - variable: - name: CLWRetFromBkg@DerivedMetaData - minvalue: 0. - maxvalue: 999. - where operator: and - assignments: - - name: CLWRetSymmetric@DerivedMetaData - type: float - function: - name: Arithmetic@ObsFunction - options: - variables: - - name: CLWRetFromObs@DerivedMetaData - - name: CLWRetFromBkg@DerivedMetaData - total coefficient: 0.5 - - # Calculate scattering index from observation - - filter: Variable Assignment - assignments: - - name: SIRetFromObs@DerivedMetaData - type: float - function: - name: SCATRetMW@ObsFunction - options: - scatret_ch238: 1 - scatret_ch314: 2 - scatret_ch890: 16 - scatret_types: [ObsValue] - - # Calculate CLW obs/bkg match index - - filter: Variable Assignment - assignments: - - name: CLWMatchIndex@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: CLWMatchIndexMW@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - clwobs_function: - name: CLWRetFromObs@DerivedMetaData - clwbkg_function: - name: CLWRetFromBkg@DerivedMetaData - clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, - 0.030, 0.030, 0.050, 0.100] - - # Calculate symmetric observation error - - filter: Variable Assignment - assignments: - - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorModelRamp@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - xvar: - name: CLWRetSymmetric@DerivedMetaData - x0: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, - 0.050, 0.100] - x1: [0.350, 0.380, 0.400, 0.450, 0.500, 1.000, 1.000, 0.000, 0.000, 0.000, - 0.000, 0.000, 0.000, 0.000, 0.000, 0.350, 0.500, 0.500, 0.500, 0.500, - 0.500, 0.500] - err0: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, - 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, - 3.000, 3.000] - err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, - 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, - 12.000, 9.000, 6.500] - - # Calculate Innovation@DerivedMetaData - - filter: Variable Assignment - assignments: - - name: Innovation@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsFunction/Arithmetic - channels: *{{observation_from_jcb}}_simulated_channels - options: - variables: - - name: brightnessTemperature@ObsValue - channels: *{{observation_from_jcb}}_simulated_channels - - name: brightnessTemperature@HofX - channels: *{{observation_from_jcb}}_simulated_channels - coefs: [1, -1] - - # Step 0-C: Assign Initial All-Sky Observation Error - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: assign error - error function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 1: Remove Observations from the Edge of the Scan - - filter: Domain Check - filter variables: - - name: brightnessTemperature - channels: 1-22 - where: - - variable: - name: MetaData/sensorScanPosition - is_in: 7-90 - actions: - - name: set - flag: ScanEdgeRemoval - - name: reject - - # Step 2: Data Thinning - - filter: Gaussian Thinning - horizontal_mesh: 145 - use_reduced_horizontal_grid: true - distance_norm: geodesic - # round_horizontal_bin_count_to_nearest: true - # partition_longitude_bins_using_mesh: true - actions: - - name: set - flag: Thinning - - name: reject - - # Step 3A: CLW Retrieval Check (observation_based) - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: CLWRetFromObs@DerivedMetaData - maxvalue: 999.0 - actions: - - name: set - flag: CLWRetrievalCheck - - name: reject - - # Step 3B: CLW Retrieval Check (background_based) - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - test variables: - - name: CLWRetFromBkg@DerivedMetaData - maxvalue: 999.0 - actions: - - name: set - flag: CLWRetrievalCheck - - name: reject - - # Step 4: Window Channel Sanity Check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-7, 16, 17-22 - test variables: - - name: Innovation@DerivedMetaData - channels: 1, 2, 5-7, 16 - maxvalue: 200.0 - minvalue: -200.0 - flag all filter variables if any test variable is out of bounds: true - actions: - - name: set - flag: WindowChannelExtremeResidual - - name: reject - - # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/HydrometeorCheckATMS - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: HydrometeorCheckATMS@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, - 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, - 3.000, 3.000, 3.000, 3.000] - clwret_function: - name: CLWRetFromObs@DerivedMetaData - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - test variables: - - name: DerivedMetaData/HydrometeorCheckATMS - channels: *{{observation_from_jcb}}_simulated_channels - maxvalue: 0.0 - actions: - - name: set - flag: HydrometeorCheck - ignore: rejected observations - - name: reject - - # Step 6: Observation Error Inflation based on Topography Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorTopo@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorTopoRad@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 7: Obs Error Inflation based on TOA Transmittancec Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorTransmitTopRad@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 8: Observation Error Inflation based on Surface Jacobian Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorSurfJacobianRad@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] - obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorSurfJacobian@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 9: Situation Dependent Check - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorFactorSituDependMW@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - clwbkg_function: - name: CLWRetFromBkg@DerivedMetaData - clwobs_function: - name: CLWRetFromObs@DerivedMetaData - scatobs_function: - name: SIRetFromObs@DerivedMetaData - clwmatchidx_function: - name: CLWMatchIndex@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, - 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, - 3.000, 3.000, 3.000, 3.000] - - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - action: - name: inflate error - inflation variable: - name: ObsErrorFactorSituDepend@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - - # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threshold - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorLat@DerivedMetaData - type: float - function: - name: ObsErrorFactorLatRad@ObsFunction - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - - - filter: Variable Assignment - assignments: - - name: ObsErrorBound@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - type: float - function: - name: ObsErrorBoundMW@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - sensor: *{{observation_from_jcb}}_sensor_id - channels: *{{observation_from_jcb}}_simulated_channels - obserr_bound_latitude: - name: ObsErrorFactorLat@DerivedMetaData - obserr_bound_transmittop: - name: ObsErrorFactorTransmitTop@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - obserr_bound_topo: - name: ObsErrorFactorTopo@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - obserr_function: - name: InitialObsError@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - threshold: 3 - threshold_precip: 2.5 - obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, - 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - - - filter: Background Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - function absolute threshold: - - name: ObsErrorBound@DerivedMetaData - channels: *{{observation_from_jcb}}_simulated_channels - actions: - - name: set - flag: GrossCheck - ignore: rejected observations - - name: reject - - # Step 11: Inter-Channel Check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - test variables: - - name: InterChannelConsistencyCheck@ObsFunction - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - use passive_bc: true - sensor: *{{observation_from_jcb}}_sensor_id - use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, - 1] - maxvalue: 1.0e-12 - actions: - - name: set - flag: InterChannelConsistency - ignore: rejected observations - - name: reject - - # Step 12: Useflag Check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *{{observation_from_jcb}}_simulated_channels - test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: *{{observation_from_jcb}}_simulated_channels - options: - channels: *{{observation_from_jcb}}_simulated_channels - use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, - 1] - minvalue: 1.0e-12 - actions: - - name: set - flag: UseflagCheck - ignore: rejected observations - - name: reject - -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 - -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject - # GeoVaLs for Driving Observation Operators (testing mode) - # -------------------------------------------------------- - geovals: - filename: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}_geoval{{atmosphere_obsdatain_suffix}}" - - # Passed benchmark for UFO testing - # -------------------------------- - passedBenchmark: 0 diff --git a/observations/atmosphere/atms_npp.yaml.j2 b/observations/atmosphere/atms_npp.yaml.j2 index 309da34..d18985c 100644 --- a/observations/atmosphere/atms_npp.yaml.j2 +++ b/observations/atmosphere/atms_npp.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] -# Cloud_Fraction: 1.0 + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_npp @@ -37,8 +37,6 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" - variables without bc: [brightnessTemperature] - channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -67,25 +65,6 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" - # ------------------------ - obs prior filters: - # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 - - filter: Variable Assignment - assignments: - - name: MetaData/zeroCloudInCRTM - type: int - function: - name: ObsFunction/Conditional - options: -# firstmatchingcase: false - defaultvalue: 0 # Will not zero clouds by default - cases: - - where: - - variable: - name: GeoVaLs/water_area_fraction - maxvalue: 0.99 - value: 1 # Will zero clouds by default - # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -350,7 +329,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *{{observation_from_jcb}}_clouds + - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -474,7 +453,7 @@ channels: *{{observation_from_jcb}}_simulated_channels # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threshold + # Remove data if abs(Obs-HofX) > absolute threhold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -508,8 +487,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threshold: 3 - threshold_precip: 2.5 + threhold: 3 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] @@ -566,116 +544,7 @@ ignore: rejected observations - name: reject -# Column liquid cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer -# Column ice cloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnIceCloud - type: float - function: - name: TotalColumnCloudGuess@ObsFunction - options: - cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer -# Combined ColumnLiquidCloud + ColumnIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnIceCloud - coefs: [1, 1] -# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnLiquidIceCloud - type: float - value: 1.0e-7 - where: - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - maxvalue: 1.0e-8 -# ratio of liquid /(Liquid+Ice) - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ColumnRatioLiquidCloud - type: float - function: - name: ObsFunction/ElementMultiply - options: - variables: - - name: DerivedMetaData/ColumnLiquidCloud - - name: DerivedMetaData/ColumnLiquidIceCloud - exponents: [1,-1] -# TotalColumnVaporGuess - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/TotalColumnVaporGuess - type: float - function: - name: ObsFunction/TotalColumnVaporGuess -# potentialTemperature at surface - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureSurface - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: surface_pressure -# potentialTemperature near 700 hPa - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - type: float - function: - name: PotentialTemperatureGuess2D@ObsFunction - options: - approximate pressure: 70000.0 -# stability = Difference between PotentialTemperatur values - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/PotentialTemperatureDiff - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: DerivedMetaData/PotentialTemperatureAt700hPa - - name: DerivedMetaData/PotentialTemperatureSurface - coefs: [1, -1] -## cao_check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: 1-7, 16-22 - where: - - variable: - name: ObsFunction/TotalColumnVaporGuess - maxvalue: 8.0 - - variable: - name: DerivedMetaData/ColumnRatioLiquidCloud - maxvalue: 0.5 - - variable: - name: DerivedMetaData/ColumnLiquidIceCloud - minvalue: 1.0e-6 - - variable: - name: DerivedMetaData/PotentialTemperatureDiff - maxvalue: 12.0 - action: - name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: From 9e069cc585e8e6b09643c350cd0cc9c7d6909119 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 22 Jan 2025 11:18:23 -0700 Subject: [PATCH 3/3] Add option for outputting post-processed soca ice increment (#64) Adds yaml section to output postprocessed ("real" aka what get added to CICE restart) sea ice increment. Part of https://github.com/NOAA-EMC/GDASApp/issues/1402 Co-authored-by: Anna Shlyaeva Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- algorithm/marine/soca_2cice_global.yaml.j2 | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/algorithm/marine/soca_2cice_global.yaml.j2 b/algorithm/marine/soca_2cice_global.yaml.j2 index 01bff96..4ccf033 100644 --- a/algorithm/marine/soca_2cice_global.yaml.j2 +++ b/algorithm/marine/soca_2cice_global.yaml.j2 @@ -29,10 +29,20 @@ variable change: sno_lev: 1 cice output: restart: {{ ice_rst }} + increment output: + datadir: Data + exp: soca2cice + type: incr + date: '{{ fcst_begin }}' + soca increment: + read_from_file: 1 + basename: ./ + ocn_filename: {{ ocn_inc }} + ice_filename: {{ ice_inc }} + date: '{{ fcst_begin }}' output variables: - sea_water_potential_temperature - sea_water_salinity - - sea_water_cell_thickness - sea_ice_area_fraction - sea_ice_thickness - sea_ice_snow_thickness @@ -47,7 +57,6 @@ states: state variables: - sea_water_potential_temperature - sea_water_salinity - - sea_water_cell_thickness - sea_ice_area_fraction - sea_ice_thickness - sea_ice_snow_thickness