Merge branch 'dev'

src/pyrad_proc/pyrad/io/read_data_sensor.py

@@ -2218,12 +2218,26 @@ def get_sensor_data(date, datatype, cfg):
 
  """
  if cfg["sensor"] == "rgage":
- datapath = cfg["smnpath"] + date.strftime("%Y%m") + "/"
- datafile = date.strftime("%Y%m%d") + "_" + cfg["sensorid"] + ".csv"
+ # Try several file formats
+ datapath1 = os.path.join(cfg["smnpath"], date.strftime("%Y%m"))
+ datapath2 = os.path.join(cfg["smnpath"], date.strftime("%Y-%m-%d"), 
+ cfg["sensorid"])
+ datafile1 = os.path.join(datapath1, date.strftime("%Y%m%d")
+ + "_" + cfg["sensorid"] + ".csv*")
+ datafile2 = os.path.join(datapath2, date.strftime("%Y%m%d")
+ + "_" + cfg["sensorid"] + ".csv*")
+
+ files1 = glob.glob(datafile1)
+ files2 = glob.glob(datafile2)
+ if len(files1):
+ datafile = files1[0]
+ if len(files2):
+ datafile = files2[0]
+
  try:
- _, sensordate, _, _, _, sensorvalue, _, _ = read_smn(datapath + datafile)
+ _, sensordate, _, _, _, sensorvalue, _, _ = read_smn(datafile)
  except Exception:
- _, sensordate, sensorvalue = read_smn2(datapath + datafile)
+ _, sensordate, sensorvalue = read_smn2(datafile)
  if sensordate is None:
  return None, None, None, None
  label = "RG"
@@ -2317,10 +2331,9 @@ def get_sensor_data(date, datatype, cfg):
 
  return sensordate, sensorvalue, label, period
 
-
 def read_smn(fname):
  """
- Reads SwissMetNet data contained in a csv file
+ Reads SwissMetNet data contained in a csv or gzipped csv file.
 
  Parameters
  ----------
@@ -2329,54 +2342,45 @@ def read_smn(fname):
 
  Returns
  -------
- smn_id, date , pressure, temp, rh, precip, wspeed, wdir : tupple
+ smn_id, date, pressure, temp, rh, precip, wspeed, wdir : tuple
  The read values
-
  """
  fill_value = 10000000.0
- try:
- with open(fname, "r", newline="") as csvfile:
- # first count the lines
- reader = csv.DictReader(csvfile)
- nrows = sum(1 for row in reader)
- smn_id = np.ma.empty(nrows, dtype="float32")
- pressure = np.ma.empty(nrows, dtype="float32")
- temp = np.ma.empty(nrows, dtype="float32")
- rh = np.ma.empty(nrows, dtype="float32")
- precip = np.ma.empty(nrows, dtype="float32")
- wspeed = np.ma.empty(nrows, dtype="float32")
- wdir = np.ma.empty(nrows, dtype="float32")
 
- # now read the data
- csvfile.seek(0)
- reader = csv.DictReader(csvfile)
- date = []
- for i, row in enumerate(reader):
- smn_id[i] = float(row["StationID"])
- date.append(datetime.datetime.strptime(row["DateTime"], "%Y%m%d%H%M%S"))
- pressure[i] = float(row["AirPressure"])
- temp[i] = float(row["2mTemperature"])
- rh[i] = float(row["RH"])
- precip[i] = float(row["Precipitation"])
- wspeed[i] = float(row["Windspeed"])
- wdir[i] = float(row["Winddirection"])
-
- pressure = np.ma.masked_values(pressure, fill_value)
- temp = np.ma.masked_values(temp, fill_value)
- rh = np.ma.masked_values(rh, fill_value)
- precip = np.ma.masked_values(precip, fill_value)
- wspeed = np.ma.masked_values(wspeed, fill_value)
- wdir = np.ma.masked_values(wdir, fill_value)
-
- # convert precip from mm/10min to mm/h
- precip *= 6.0
-
- csvfile.close()
-
- return smn_id, date, pressure, temp, rh, precip, wspeed, wdir
- except EnvironmentError as ee:
- warn(str(ee))
- warn("Unable to read file " + fname)
+ try:
+ # Use pandas to read the file (supports .gz files directly)
+ df = pd.read_csv(fname, compression='gzip' if fname.endswith('.gz') else None)
+
+ # Convert date strings to datetime objects
+ df['DateTime'] = pd.to_datetime(df['DateTime'], format='%Y%m%d%H%M%S')
+
+ # Identify columns by searching for keywords (handles cases like AirPressure:degC)
+ air_pressure_col = [col for col in df.columns if 'AirPressure' in col][0]
+ temp_col = [col for col in df.columns if '2mTemperature' in col][0]
+ rh_col = [col for col in df.columns if 'RH' in col][0]
+ precip_col = [col for col in df.columns if 'Precipitation' in col][0]
+ wspeed_col = [col for col in df.columns if 'Windspeed' in col][0]
+ wdir_col = [col for col in df.columns if 'Winddirection' in col][0]
+
+ # Mask invalid data (fill_value)
+ pressure = np.ma.masked_values(df[air_pressure_col].astype('float32'), fill_value)
+ temp = np.ma.masked_values(df[temp_col].astype('float32'), fill_value)
+ rh = np.ma.masked_values(df[rh_col].astype('float32'), fill_value)
+ precip = np.ma.masked_values(df[precip_col].astype('float32'), fill_value)
+ wspeed = np.ma.masked_values(df[wspeed_col].astype('float32'), fill_value)
+ wdir = np.ma.masked_values(df[wdir_col].astype('float32'), fill_value)
+
+ # Convert precip from mm/10min to mm/h
+ precip *= 6.0
+
+ # Extract smn_id and date
+ smn_id = df['StationID'].astype('float32').values
+ date = df['DateTime'].tolist()
+
+ return smn_id, date, pressure, temp, rh, precip, wspeed, wdir
+
+ except Exception as e:
+ warn(f"Unable to read file {fname}: {e}")
  return None, None, None, None, None, None, None, None
 
 

src/pyrad_proc/pyrad/proc/process_Doppler.py

@@ -65,7 +65,9 @@ def process_turbulence(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain,
+ "dBuZ" or "dBZ" or "dBZc" or "dBuZv" or "dBZv" or "dBZvc" or "CNRc", and, 
+ "W" or "Wv" or "Wu" or "Wvu" or "WD" or "WDc"
  radius : float. Dataset keyword
  Search radius for calculating Eddy Dissipation Rate (EDR).
  Default 2
@@ -97,7 +99,7 @@ def process_turbulence(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field "EDR"
  ind_rad : int
  radar index
 
@@ -235,7 +237,8 @@ def process_dealias_fourdd(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain
+ "V" or "Vc"
  filt : int. Dataset keyword
  Flag controlling Bergen and Albers filter, 1 = yes, 0 = no.
  sign : int. Dataset keyword
@@ -252,7 +255,7 @@ def process_dealias_fourdd(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field "dealV" or "dealVc" (if Vc was provided)
  ind_rad : int
  radar index
 
@@ -357,7 +360,8 @@ def process_dealias_region_based(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain,
+ "V" or "Vc"
  interval_splits : int, optional
  Number of segments to split the nyquist interval into when finding
  regions of similar velocity. More splits creates a larger number
@@ -387,7 +391,7 @@ def process_dealias_region_based(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field "dealV" or "dealVc" (if Vc was provided)
  ind_rad : int
  radar index
 
@@ -458,7 +462,8 @@ def process_dealias_unwrap_phase(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain,
+ "V" or "Vc"
  unwrap_unit : {'ray', 'sweep', 'volume'}, optional
  Unit to unwrap independently. 'ray' will unwrap each ray
  individually, 'sweep' each sweep, and 'volume' will unwrap the
@@ -473,7 +478,7 @@ def process_dealias_unwrap_phase(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field "dealV" or "dealVc" (if Vc was provided)
  ind_rad : int
  radar index
 
@@ -534,7 +539,10 @@ def process_radial_velocity(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain
+ WIND_SPEED, and,
+ WIND_DIRECTION, and,
+ wind_vel_v
  latitude, longitude : float
  arbitrary coordinates [deg] from where to compute the radial
  velocity. If any of them is None it will be the radar position
@@ -547,7 +555,7 @@ def process_radial_velocity(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field "V"
  ind_rad : int
  radar index
 
@@ -665,7 +673,8 @@ def process_wind_vel(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain
+ "V" or "Vc"
  vert_proj : Boolean
  If true the vertical projection is computed. Otherwise the
  horizontal projection is computed
@@ -675,7 +684,9 @@ def process_wind_vel(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field
+ "wind_vel_h_az", (if vert_proj is False), or,
+ "wind_vel_v" (if vert_proj is True)
  ind_rad : int
  radar index
 
@@ -737,7 +748,7 @@ def process_windshear(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field "windshear_v"
  ind_rad : int
  radar index
 
@@ -786,7 +797,8 @@ def process_windshear_lidar(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain
+ "V" or "Vc"
  az_tol : float
  The tolerance in azimuth when looking for gates on top
  of the gate when computation is performed
@@ -797,7 +809,7 @@ def process_windshear_lidar(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output field "windshear_v"
  ind_rad : int
  radar index
 
@@ -847,14 +859,17 @@ def process_vad(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain
+ "V" or "Vc"
  radar_list : list of Radar objects
  Optional. list of radar objects
 
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output fields
+ "wind_vel_h_u", "wind_vel_h_v", "wind_vel_v",
+ "estV", "stdV", and "diffV"
  ind_rad : int
  radar index
 
@@ -923,7 +938,9 @@ def process_dda(procstatus, dscfg, radar_list=None):
  data set configuration. Accepted Configuration Keywords::
 
  datatype : string. Dataset keyword
- The input data type
+ The input data type, must contain
+ "V" or "Vc", and,
+ "dBuZ", "dBZ", or "dBZc"
 
  gridconfig : dictionary. Dataset keyword
  Dictionary containing some or all of this keywords:
@@ -997,7 +1014,8 @@ def process_dda(procstatus, dscfg, radar_list=None):
  Returns
  -------
  new_dataset : dict
- dictionary containing the output
+ dictionary containing the output fields
+ "wind_vel_h_u", "wind_vel_h_v" and "wind_vel_v"
  ind_rad : int
  radar index