Merge Data V6 Branch

data_prep/configs/ursa.yaml

@@ -23,8 +23,9 @@ data types:
 
   - name: raw_surface_obs
     type: tank
-    num_tasks: 4
+    num_tasks: 1
     batch_days: 30
+    memory: 48G
     mapping: bufr_surface_obs_raw.py
     paths:
       prepbufr:
@@ -57,7 +58,7 @@ data types:
   - name: raw_radiosonde
     type: tank
     num_tasks: 1
-    batch_days: 30
+    batch_days: 15
     mapping: bufr_radiosonde_raw.py
     paths:
       prepbufr:
@@ -77,6 +78,7 @@ data types:
         - 18/atmos/gdas.t18z.uprair.tm00.bufr_d
 
   - name: ssmis
+    memory: 128G
     type: tank
     num_tasks: 4
     batch_days: 30
@@ -87,6 +89,24 @@ data types:
       - 12/atmos/gdas.t12z.ssmisu.tm00.bufr_d
       - 18/atmos/gdas.t18z.ssmisu.tm00.bufr_d
 
+  - name: seviri
+    memory: 128G
+    type: tank
+    num_tasks: 4
+    batch_days: 30
+    mapping: bufr_seviri.py
+    paths:
+      sevcsr:
+        - 00/atmos/gdas.t00z.sevcsr.tm00.bufr_d
+        - 06/atmos/gdas.t06z.sevcsr.tm00.bufr_d
+        - 12/atmos/gdas.t12z.sevcsr.tm00.bufr_d
+        - 18/atmos/gdas.t18z.sevcsr.tm00.bufr_d
+      sevasr:
+        - 00/atmos/gdas.t00z.sevasr.tm00.bufr_d
+        - 06/atmos/gdas.t06z.sevasr.tm00.bufr_d
+        - 12/atmos/gdas.t12z.sevasr.tm00.bufr_d
+        - 18/atmos/gdas.t18z.sevasr.tm00.bufr_d
+
   - name: amsua
     type: tank
     num_tasks: 4
@@ -98,6 +118,17 @@ data types:
       - 12/atmos/gdas.t12z.1bamua.tm00.bufr_d
       - 18/atmos/gdas.t18z.1bamua.tm00.bufr_d
 
+  - name: ascat
+    type: tank
+    num_tasks: 16
+    batch_days: 30
+    mapping: bufr_ascat.py
+    paths:
+      - 00/atmos/gdas.t00z.ascatt.tm00.bufr_d
+      - 06/atmos/gdas.t06z.ascatt.tm00.bufr_d
+      - 12/atmos/gdas.t12z.ascatt.tm00.bufr_d
+      - 18/atmos/gdas.t18z.ascatt.tm00.bufr_d
+
   - name: viirs
     type: tank
     num_tasks: 24
@@ -111,8 +142,9 @@ data types:
 
   - name: avhrr
     type: tank
-    num_tasks: 24
-    batch_days: 15
+    num_tasks: 64
+    batch_days: 7
+    memory: 128G
     mapping: bufr_avhrr.py
     paths:
       am:

data_prep/mapping/bufr_amsua.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
 import os
+
 from bufr.obs_builder import ObsBuilder, add_main_functions, map_path
 
+
 MAPPING_PATH = map_path('bufr_amsua.yaml')
 
 

data_prep/mapping/bufr_ascat.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 
 import os
+
 from bufr.obs_builder import ObsBuilder, add_main_functions, map_path
 
-MAPPING_PATH = map_path('bufr_scatwnd_ascat.yaml')
+
+MAPPING_PATH = map_path('bufr_ascat.yaml')
 
 
 class BufrAscatObsBuilder(ObsBuilder):

data_prep/mapping/bufr_ascat.yaml

@@ -13,74 +13,86 @@ bufr:
         minute: "*/MINU"
         second: "*/SECO"
 
-    dataReceiptTime:
-      datetime:
-        year: "*/RCYR"
-        month: "*/RCMO"
-        day: "*/RCDY"
-        hour: "*/RCHR"
-        minute: "*/RCMI"
-
     latitude:
-      query: "*/CLAT"
+      query: "*/CLATH"
 
     longitude:
-      query: "*/CLON"
+      query: "*/CLONH"
 
     satelliteId:
       query: "*/SAID"
 
-    qualityFlags:
-      query: "*/WVCQ"
+    backscatter:
+      query: "*/ASCATSG0/BKST"
+
+    beamId:
+      query: "*/ASCATSG0/BEAMI"
+
+    radarIncidenceAngle:
+      query: "*/ASCATSG0/RAIA"
 
-    # ObsValue - Wind Direction
-    windDirectionAt10M:
-      query: '*/WD10'
-      type: float
+    antennaBeamAzimuthAngle:
+      query: "*/ASCATSG0/ANAZ"
 
-    # ObsValue - Wind Speed
-    windSpeedAt10M:
-      query: '*/WS10'
+    ascatKpEstimateQuality:
+      query: "*/ASCATSG0/AKPEQ"
+
+    ascatSigma0Usability:
+      query: "*/ASCATSG0/ASGU"
 
 encoder:
+
+  dimensions:
+    - name: ch
+      path: "*/ASCATSG0"
+
   variables:
     # MetaData
-    - name: "MetaData/time"
-      source: variables/timestamp
+    - name: time
+      source: timestamp
       longName: "Observation Time"
       units: "seconds since 1970-01-01T00:00:00Z"
 
-    - name: "MetaData/dataReceiptTime"
-      source: variables/dataReceiptTime
-      longName: "Observation Receipt Time"
-      units: "seconds since 1970-01-01T00:00:00Z"
-
-    - name: "MetaData/latitude"
-      source: variables/latitude
+    - name: latitude
+      source: latitude
       longName: "Latitude"
       units: "degrees_north"
       range: [ -90, 90 ]
 
-    - name: "MetaData/longitude"
-      source: variables/longitude
+    - name: longitude
+      source: longitude
       longName: "Longitude"
       units: "degrees_east"
       range: [ -180, 180 ]
 
-    - name: "MetaData/satelliteIdentifier"
-      source: variables/satelliteId
+    - name: satelliteIdentifier
+      source: satelliteId
       longName: "Satellite Identifier"
 
-    - name: "MetaData/qualityFlags"
-      source: variables/qualityFlags
-      longName: "Quality Flags"
+    - name: backscatter
+      source: backscatter
+      longName: "Backscatter"
+
+    - name: beamId
+      source: beamId
+      longName: "Beam Identifier"
 
-    - name: "ObsValue/windDirection"
-      source: variables/windDirectionAt10M
-      longName: "10-meter Wind Direction"
+    - name: radarIncidenceAngle
+      source: radarIncidenceAngle
+      longName: "Radar Incidence Angle"
       units: "degree"
+      range: [ 0, 90 ]
+
+    - name: antennaBeamAzimuthAngle
+      source: antennaBeamAzimuthAngle
+      longName: "Antenna Beam Azimuth Angle"
+      units: "degree"
+      range: [ 0, 360 ]
+
+    - name: ascatKpEstimateQuality
+      source: ascatKpEstimateQuality
+      longName: "ASCAT Kp Estimate Quality"
 
-    - name: "ObsValue/windSpeed"
-      source: variables/windSpeedAt10M
-      longName: "10-meter Wind Speed"
-      units: "m s-1"
+    - name: ascatSigma0Usability
+      source: ascatSigma0Usability
+      longName: "ASCAT Sigma0 Usability"

data_prep/mapping/bufr_radiosonde.py

@@ -32,7 +32,7 @@ def make_obs(self, comm, input_path):
         self._apply_quality_flag(container, 'northwardWind', 'windQuality')
         self._apply_quality_flag(container, 'eastwardWind',  'windQuality')
         self._apply_quality_flag(container, 'airPressure', 'airPressureQuality')
-        self._apply_quality_flag(container, 'height', 'heightQuality')
+        self._apply_quality_flag(container, 'height_prepbufr', 'heightQuality')
 
         # Add timestamps
         reference_time = self._get_reference_time(input_path)

data_prep/mapping/bufr_radiosonde.yaml

@@ -1,5 +1,5 @@
 bufr:
-  group_by_variable: height
+  group_by_variable: height_prepbufr
   subsets:
     - ADPUPA
   variables:
@@ -43,7 +43,7 @@ bufr:
     airPressure:
       query: "*/PRSLEVEL/P___INFO/P__EVENT{1}/POB"
 
-    height:
+    height_prepbufr:
       query: "*/PRSLEVEL/Z___INFO/Z__EVENT{1}/ZOB"
 
     # Quality Markers
@@ -83,8 +83,8 @@ encoder:
       source: stationIdentification
       longName: "Station Identification"
 
-    - name: "stationElevation"
-      source: stationElevation
+    - name: "stationElevation_prepbufr"
+      source: stationElevation_prepbufr
       longName: "Station Elevation"
       units: "meters"
       range: [-100, 5000]
@@ -130,8 +130,8 @@ encoder:
       units: "hPa"
       range: [0, 1200]
 
-    - name: "height"
-      source: height
+    - name: "height_prepbufr"
+      source: height_prepbufr
       longName: "Height"
       units: "meters"
       range: [0, 60000]

data_prep/mapping/bufr_radiosonde_adpupa.yaml

@@ -23,6 +23,10 @@ bufr:
     reportId:
       query: "*/RPID"
 
+    height_bufr:
+      query: "*/SELV"
+      type: float
+
     airPressure:
       query: "[*/UARLV/PRLC, */PRLC]"  # Air pressure at station level
       type: float
@@ -52,6 +56,11 @@ encoder:
   variables:
     # MetaData
 
+    - name: "height_bufr"
+      source: height_bufr
+      longName: "height"
+      units: "m"
+
     - name: "airTemperature"
       source: airTemperature
       longName: "Air Temperature"
@@ -64,7 +73,7 @@ encoder:
       units: "deg C"
       range: [ -100, 100 ]
 
-    - name: "ralativeHumidity"
+    - name: "relativeHumidity"
       source: relativeHumidity
       longName: "Specific Humidity"
       units: "mg/kg"

data_prep/mapping/bufr_radiosonde_prepbufr.yaml

@@ -15,6 +15,9 @@ bufr:
       transforms:
         - wrap: [ -180.0, 180.0 ]
 
+    stationElevation:
+      query: "*/ELV"
+
     sequenceId:
       query: "*/SID"
 
@@ -41,7 +44,7 @@ bufr:
     airPressureProgramCode:
       query: "*/PRSLEVEL/P___INFO/P__EVENT{1}/PPC"
 
-    height:
+    height_prepbufr:
       query: "*/PRSLEVEL/Z___INFO/Z__EVENT{1}/ZOB"
 
     # Quality Markers

data_prep/mapping/bufr_radiosonde_raw.py

@@ -29,6 +29,9 @@ def __init__(self):
 
     # Override
     def make_obs(self, comm, input_dict) -> bufr.DataContainer:
+        if PrepbufrKey not in input_dict or LowResDumpKey not in input_dict:
+            return bufr.DataContainer()
+
         prep_container = bufr.Parser(input_dict[PrepbufrKey], self.map_dict[PrepbufrKey]).parse(comm)
         prep_container.apply_mask(~prep_container.get('launchCycleTime').mask)
         prep_container.apply_mask(~prep_container.get('driftLatitude').mask)
@@ -120,14 +123,16 @@ def _process_dump(self, comm, input_dict, prep_container, data_key) -> bufr.Data
         for var in container.list():
             data = container.get(var)
             path = container.get_paths(var)
-            matched_data = np.array([data[dump_idx] for dump_idx, prep_idx, flight_idx in matching_idxs])
+            idxs = np.array([dump_idx for dump_idx, prep_idx, flight_idx in matching_idxs])
+            matched_data = data[idxs]
             new_container.add(var, matched_data, path)
 
         # Add the prepbufr data to the new container
         for var in ['driftTime',
                     'driftLatitude',
                     'driftLongitude',
-                    'height',
+                    'height_prepbufr',
+                    'stationElevation',
                     'airTemperatureQuality',
                     'specificHumidityQuality',
                     'dewPointTemperatureQuality',
@@ -137,7 +142,9 @@ def _process_dump(self, comm, input_dict, prep_container, data_key) -> bufr.Data
 
             data = prep_container.get(var)
             path = prep_container.get_paths(var)
-            matched_data = np.array([data[prep_idx] for dump_idx, prep_idx, flight_idx in matching_idxs])
+            idxs = np.array([prep_idx for dump_idx, prep_idx, flight_idx in matching_idxs])
+            matched_data = data[idxs]
+
             if matched_data.dtype == np.dtype('float64'):
                 matched_data = matched_data.astype('float32')
             new_container.add(var, matched_data, path)
@@ -174,11 +181,17 @@ def _make_description(self):
                 'units': "degree_east"
             },
             {
-                'name': "height",
-                'source': 'height',
+                'name': "height_prepbufr",
+                'source': 'height_prepbufr',
                 'longName': "Height",
                 'units': "meters"
             },
+            {
+                'name': "stationElevation",
+                'source': 'stationElevation',
+                'longName': "Station Elevation",
+                'units': "meters"
+            },
             {
                 'name': "airTemperatureQuality",
                 'source': 'airTemperatureQuality',