Add capability to create varobs and cx files for MTG-IRS (#234)

* namelists for mtgirs

* add ops code to include mtgirs for cx and varobs files

* add MTG-IRS ctests

* add tests to CMakeLists file

* add MTG-IRS

* to retrigger CI

* retrigger CI

* test with older version of JCSDA container

.github/workflows/ci.yml

@@ -75,7 +75,7 @@ jobs:
             --entrypoint=/usr/local/src/${REPO}/${PR}/build-and-test \
             --workdir=/usr/local/src/${REPO}/${PR} \
             --volume ${PWD}/${PR}:/usr/local/src/${REPO}/${PR} \
-            'jcsda/docker-gnu-openmpi-dev:latest'
+            'jcsda/docker-gnu-openmpi-dev:skylab-v8'
 
   build2:
     if: |

deps/ops/CMakeLists.txt

@@ -20,7 +20,6 @@ set(OPS_SOURCE_FILES
   code/OpsMod_AirQuality/OpsMod_AirQuality.f90
   code/OpsMod_Argument/OpsMod_Argument.f90
   code/OpsMod_CXInfo/OpsMod_CXInfo.f90
-  code/OpsMod_Control/OpsMod_Control.f90
   code/OpsMod_DateTime/OpsMod_DateTime.f90
   code/OpsMod_GatherSpread/OpsMod_GatherSpread.F90
   code/OpsMod_HorizontalInterp/OpsMod_HorizontalInterp.f90
@@ -88,6 +87,7 @@ set(OPS_SOURCE_FILES
   stubs/OpsMod_Varobs/OpsMod_Varobs.f90
   stubs/Ops_SatRad_Info/OpsMod_SatRad_RTmodel.f90
   stubs/Ops_SatRad_SetUp/OpsMod_SatRad_SetUp.f90
+  stubs/OpsMod_Control/OpsMod_Control.f90
 )
 
 # Intel inserts lines in the source during pre-processing that contain the source file path, if the pathname

deps/ops/code/OpsMod_Control/Ops_ReadObsGroupSwitches.inc → deps/ops/stubs/OpsMod_Control/Ops_ReadObsGroupSwitches.inc

@@ -102,6 +102,7 @@ Using(ObsGroupHIRAS) = ops_env_is_true ("OPS_HIRAS")
 Using(ObsGroupOceanWinds) = ops_env_is_true ("OPS_OCEANWINDS")
 Using(ObsGroupGIIRSLW) = ops_env_is_true ("OPS_GIIRSLW")
 Using(ObsGroupGIIRSMW) = ops_env_is_true ("OPS_GIIRSMW")
+Using(ObsGroupMTGIRS) = ops_env_is_true ("OPS_MTGIRS")
 
 ALLOCATE (obs_group_list(COUNT (using)))
 obs_group_list = IMDI

deps/ops/stubs/OpsMod_Varobs/Ops_GetDefaultVarfields.inc

@@ -177,6 +177,15 @@ SELECT CASE (ObsGroup)
     IF (.NOT. RTTOV_CloudSwitch) THEN
       Varfields(13:14) = (/Varfield_cloudtopp, Varfield_cloudfrac/)
     END IF
+  CASE (ObsGroupMTGIRS)
+    Varfields(1:12) = (/Varfield_tskin,Varfield_tcozone,Varfield_satzenith, &
+                        Varfield_surface,Varfield_stratt,Varfield_satid, &
+                        Varfield_numchans,Varfield_channum,Varfield_britemp, &
+                        Varfield_emissivity,Varfield_solzenith, &
+                        Varfield_qcinfo/)
+    IF (.NOT. RTTOV_CloudSwitch) THEN
+      Varfields(13:14) = (/Varfield_cloudtopp, Varfield_cloudfrac/)
+    END IF
   CASE (ObsGroupIN3DIClr)
     Varfields(1:12) = (/Varfield_britemp,Varfield_satid,Varfield_satzenith, &
                         Varfield_solzenith,Varfield_tskin,Varfield_tcozone, &

deps/ops/stubs/OpsMod_Varobs/Ops_SetupVarobsLevDepC.inc

@@ -74,7 +74,8 @@ USE OpsMod_ObsGroupInfo, ONLY: &
     ObsGroupHIRAS,             &
     ObsGroupOceanWinds,        &
     ObsGroupGIIRSLW,           &
-    ObsGroupGIIRSMW
+    ObsGroupGIIRSMW,           &
+    ObsGroupMTGIRS
 
 USE OpsMod_ObsInfo, ONLY: &
     OB_type
@@ -166,6 +167,11 @@ SELECT CASE (Observations % header % ObsGroup)
     ObsLevelType = 0.0
     NumLevelLevs = 1
 
+  CASE (ObsGroupMTGIRS)
+
+    ObsLevelType = 0.0
+    NumLevelLevs = 1
+
   CASE (ObsGroupAMSUB)
 
     ObsLevelType = 0.0

deps/ops/stubs/Ops_Constants/OpsFn_IsSatRadGroup.inc

@@ -42,7 +42,8 @@ IF (obsgroup == ObsGroupABIClr .OR. &
     obsgroup == ObsGroupGMIhigh .OR. &
     obsgroup == ObsGroupSSMIS .OR. &
     obsgroup == ObsGroupGIIRSLW .OR. &
-    obsgroup == ObsGroupGIIRSMW) THEN
+    obsgroup == ObsGroupGIIRSMW .OR. &
+    obsgroup == ObsGroupMTGIRS) THEN
 
   OpsFn_IsSatRadGroup = .TRUE.
 ELSE

deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNameToNum.inc

@@ -128,6 +128,8 @@ SELECT CASE (ObsGroupName)
     OpsFn_ObsGroupNameToNum = ObsGroupSurface
   CASE ("SurfaceCloud")
     OpsFn_ObsGroupNameToNum = ObsGroupSurfaceCloud
+  CASE ("MTGIRS")
+    OpsFn_ObsGroupNameToNum = ObsGroupMTGIRS
   CASE DEFAULT
     OpsFn_ObsGroupNameToNum = IMDI
 END SELECT

deps/ops/stubs/Ops_Constants/OpsFn_ObsGroupNumToName.inc

@@ -125,6 +125,8 @@ SELECT CASE (ObsGroup)
     OpsFn_ObsGroupNumToName = "Surface"
   CASE (ObsGroupSurfaceCloud)
     OpsFn_ObsGroupNumToName = "SurfaceCloud"
+  CASE (ObsGroupMTGIRS)
+    OpsFn_ObsGroupNumToName = "MTGIRS"
   CASE DEFAULT
     OpsFn_ObsGroupNumToName = "Unknown"
 END SELECT

deps/ops/stubs/Ops_Constants/OpsMod_ObsGroupInfo.f90

@@ -73,7 +73,10 @@ MODULE OpsMod_ObsGroupInfo
 INTEGER, PARAMETER :: ObsGroupGIIRSLW      =  62 ! => GIIRS LW radiances
 INTEGER, PARAMETER :: ObsGroupGIIRSMW      =  63 ! => GIIRS MW radiances
 INTEGER, PARAMETER :: ObsGroupScatwindChosen =  64 ! => Scatterometer winds (single solution)
-INTEGER, PARAMETER :: max_obs_group_num    =  64 ! Number of observation groups
+INTEGER, PARAMETER :: ObsGroupSurfaceLSDA    =  65 ! => LSDA (Surface)
+INTEGER, PARAMETER :: ObsGroupAscatLSDA      =  66 ! => LSDA (ASCAT)
+INTEGER, PARAMETER :: ObsGroupMTGIRS         =  67 ! => MTG-IRS radiances
+INTEGER, PARAMETER :: max_obs_group_num      =  67 ! Number of observation groups
 
 CONTAINS
 

deps/update_ops_sources.sh

@@ -29,7 +29,6 @@ cp --parents --recursive code/GenMod_ModelIO "$DEST_DIR"
 cp --parents --recursive code/OpsMod_AirQuality "$DEST_DIR"
 cp --parents --recursive code/OpsMod_Argument "$DEST_DIR"
 cp --parents --recursive code/OpsMod_CXInfo "$DEST_DIR"
-cp --parents --recursive code/OpsMod_Control "$DEST_DIR"
 cp --parents --recursive code/OpsMod_DateTime "$DEST_DIR"
 cp --parents --recursive code/OpsMod_GatherSpread "$DEST_DIR"
 cp --parents --recursive code/OpsMod_HorizontalInterp "$DEST_DIR"

etc/global/cx/MTGIRS.nl

@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=4,10,12,24,31,33,254,266,267,268,407,409,3209,3210,3236,3245,16222
+/

etc/global/varobs/MTGIRS.nl

@@ -0,0 +1,3 @@
+&VarobsControlNL
+Varfields=10,11,18,19,21,28,31,35,36,54,55,57,80
+/

test/CMakeLists.txt

@@ -351,6 +351,10 @@ ADD_WRITER_TEST(NAME     varobswriter_globalnamelist_seviriclr
                 YAML     varobswriter_globalnamelist_seviriclr.yaml
                 NAMELIST ../../etc/global/varobs/SEVIRIClr.nl
                 DATA     varobs_globalnamelist_seviriclr.nc4)
+ADD_WRITER_TEST(NAME     varobswriter_globalnamelist_mtgirs
+                YAML     varobswriter_globalnamelist_mtgirs.yaml
+                NAMELIST ../../etc/global/varobs/MTGIRS.nl
+                DATA     varobs_globalnamelist_mtgirs.nc4)
 ADD_WRITER_TEST(NAME     varobswriter_globalnamelist_sonde
                 YAML     varobswriter_globalnamelist_sonde.yaml
                 NAMELIST ../../etc/global/varobs/Sonde.nl
@@ -665,6 +669,10 @@ ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_seviriclr
                 YAML     cxwriter_globalnamelist_seviriclr.yaml
                 NAMELIST ../../etc/global/cx/SEVIRIClr.nl
                 DATA     cx_globalnamelist_seviriclr.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_mtgirs
+                YAML     cxwriter_globalnamelist_mtgirs.yaml
+                NAMELIST ../../etc/global/cx/MTGIRS.nl
+                DATA     cx_globalnamelist_mtgirs.nc4 dummy.nc4)
 ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_sonde
                 YAML     cxwriter_globalnamelist_sonde.yaml
                 NAMELIST ../../etc/global/cx/Sonde.nl

test/generate_unittest_netcdfs.py

@@ -879,6 +879,18 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
                                   'ObsValue/cloud_amount', 'ObsValue/cloud_top_temperature', 'ObsValue/lower_layer_pressure'],
                                  ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'],
                                  'testinput/varobs_ukvnamelist_seviriasr.nc4')
+
+    # MTG-IRS
+    output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude',
+                                  'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle',
+                                  'MetaData/solarZenithAngle', 'OutputToVAR/pressureAtTopOfCloud', 'OneDVar/cloudAmount',
+                                  'MetaData/ozoneTotal'],
+                                 ['ObsValue/radiance', 'DerivedObsValue/brightnessTemperature', 'EffectiveError/brightnessTemperature',
+                                  'OneDVar/emissivity', 'BiasCorrObsValue/brightnessTemperature',
+                                  'thickness_850_300hPa_satid_13Predictor/brightnessTemperature',
+                                  'thickness_850_300hPa_satid_17Predictor/brightnessTemperature'],
+                                 ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier', 'MetaData/observationSubTypeNum'],
+                                 'testinput/varobs_globalnamelist_mtgirs.nc4')
 
     # Aircraft
     output_full_varobs_to_netcdf(['MetaData/latitude',
@@ -1113,6 +1125,14 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
                               'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels',
                               'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
                              'testinput/cx_ukvnamelist_seviriasr.nc4')
+
+    # MTG-IRS
+    output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m',
+                              'northward_wind_at_10m', 'air_temperature_at_2m', 'relative_humidity_at_2m', 'air_pressure_at_sea_level'],
+                             ['air_potential_temperature', 'water_vapor_mixing_ratio_wrt_moist_air', 'cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water',
+                              'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water', 'air_pressure_levels',
+                              'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
+                             'testinput/cx_globalnamelist_mtgirs.nc4')
 
     # GroundGPS
     output_full_cx_to_netcdf(['skin_temperature_at_surface', 'ice_area_fraction', 'height_above_mean_sea_level_at_surface', 'air_pressure_at_surface', 'eastward_wind_at_10m',

test/testinput/cxwriter_globalnamelist_mtgirs.yaml

@@ -0,0 +1,58 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: MTGIRS
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/varobs_globalnamelist_mtgirs.nc4
+      observed variables: [radiance]
+      simulated variables: [brightnessTemperature]
+      derived variables: [brightnessTemperature]
+      channels: &all_channels 1, 3
+    geovals:
+      filename: Data/cx_globalnamelist_mtgirs.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the VarObsFile.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    ### Assign nominal error (ensure not missing values) ###
+    - filter: Perform Action
+      filter variables:
+      - name: brightnessTemperature
+        channels: *all_channels
+      action:
+        name: assign error
+        error parameter: 10.0
+    - filter: BlackList
+      filter variables:
+      - name: brightnessTemperature
+        channels: *all_channels
+      where:
+      - variable:
+          name: MetaData/latitude #ensures ob2 is failed
+        value: is_not_valid
+    - filter: Cx Writer
+      variables_for_quality_control:
+      - name: brightnessTemperature
+        channels: *all_channels
+      namelist_directory: ../etc/global/cx
+      reject_obs_with_all_variables_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["1","2","3","4","5","6","13","16","17"]
+      expected_upper_air_variables: ["1","5","11","29","30","31","34","35"]
+      expected_main_table_columns:
+      - # batch 1 - 9 1dvalues first ; then 2d values in expected order
+        # observation 3 is rejected by the tests above hence only 3 (1,2,4) columns
+        - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"]  # column 1 - 1st observation
+        - ["27.30","37.30","67.30","77.30","47.30","57.30","7.30","87.30","17.30","3.10","3.20","3.30","13.10","13.20","13.30","43.10","43.20","43.30","23.10","23.20","23.30","33.10","33.20","33.30","53.10","53.20","53.30","73.10","73.20","73.30","63.10","63.20","63.30"]  # column 3 - 3rd observation
+        - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column 4 - 4th observation
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0