[f] attenuation and entities of 15min-steps only

8e3f5782 · Fabian Kovac · a03d3467 · 8e3f5782
Commit 8e3f5782 authored Oct 22, 2021 by Fabian Kovac
--- a/FHSTP/01_data_preparation/prep.py
+++ b/FHSTP/01_data_preparation/prep.py
@@ -2,8 +2,8 @@
 # Title: Data Preparation for LINK Configs and Transmissions
 # Author: Fabian Kovac <ds191008@fhstp.ac.at>
 # Team: University of Applied Sciences St. Pölten
-# Version: 2.9
-# Last changed: 2021-10-06
+# Version: 3.0
+# Last changed: 2021-10-22
 #

 import sys
@@ -142,7 +142,7 @@ def get_midpoint(lon_a: np.array, lat_a: np.array, lon_b: np.array, lat_b: np.ar


 def get_distance(lon_a: np.array, lat_a: np.array, alt_a: np.array, lon_b: np.array, lat_b: np.array, alt_b: np.array) -> np.array:
-    """Calculcates distance between two coordinates in m using a WGS84 rotation-ellipsoid
+    """Calculcates distance between two coordinates in km using a WGS84 rotation-ellipsoid
    
    Parameters:
        lon_a (np.array): Longitudes of point A
@@ -153,13 +153,13 @@ def get_distance(lon_a: np.array, lat_a: np.array, alt_a: np.array, lon_b: np.ar
        alt_b (np.array): Altitudes of point B
    
    Returns:
-        length (np.array): Vector with distances in m
+        length (np.array): Vector with distances in km
    """
    
-    # constants (equator radius and pole radius in m)
+    # constants (equator radius and pole radius in km)
    # r_equator is the 'semi-major axis' and r_pole the 'semi-minor axis' on a WGS84 ellipsoid
-    r_equator = 6378137
-    r_pole = 6356752.3142
+    r_equator = 6378.137
+    r_pole = 6356.7523142
    
    # calculate points on rotation-ellipsoid
    # point A
@@ -307,7 +307,7 @@ def get_inca_data(datetimes: np.array, lon_a: np.array, lat_a: np.array, lon_b:
    # define n INCA RR points for each km of link length
    # --> min. points: 3 (start, mid and end) if length < 3km
    # --> get odd amount of RR points to always include the middle of the link
-    n = np.round(lengths/1000, decimals = 0).astype(int)
+    n = np.round(lengths, decimals = 0).astype(int)
    n = np.where(n < 3, 3, n)
    n = np.where(n%2 == 0, n+1, n)
    
@@ -426,6 +426,7 @@ def prep() -> None:
        df_altitudes = pd.read_csv('files_from_H3A/CONFIG_ALTITUDES.csv', sep = ';')
        df_altitudes = df_altitudes[['LINKID', 'ANT_HEIGHT_ALTITUDE_A', 'ANT_HEIGHT_ALTITUDE_B']]
        df_config = pd.merge(df_config, df_altitudes, how = 'inner', left_on = 'LINKID', right_on = 'LINKID')
+        del df_altitudes
        _log('Added altitude of links')
    
    
@@ -438,7 +439,7 @@ def prep() -> None:
    df_config['ANT_HEIGHT_ALTITUDE_MID'] = df_config['ANT_HEIGHT_ALTITUDE_MID'].astype('int')
    _log('Calculated midpoint of links')
    
-    # calculate LENGTH in m between links
+    # calculate LENGTH in km between links
    df_config['LENGTH'] = get_distance(
        df_config['LONGITUDE_A'],
        df_config['LATITUDE_A'],
@@ -486,40 +487,41 @@ def prep() -> None:
    df_link['BEGINTIME'] = df_link['BEGINTIME'].dt.tz_localize('Europe/Vienna').dt.tz_convert('UTC').dt.tz_localize(None)
    _log('Converted BEGINTIME to UTC')
    
-    # fix 15min timelag in transmissions
-    df_link['BEGINTIME'] = df_link['BEGINTIME'] - pd.Timedelta(15, unit = 'min')
-    _log('Fixed 15min timelag in transmissions')
-    
    # only use transmissions with begintime matching INCA date
    date_inca = np.datetime64(f'{str(dir_inca.stem)[0:4]}-{str(dir_inca.stem)[-4:-2]}-{str(dir_inca.stem)[-2:]}')
    df_link = df_link[df_link['BEGINTIME'].dt.date == date_inca]
    _log('Filtered transmissions to match BEGINTIME with INCA date')
    
    
-    # copy REMOTERXLEVEL to PMIN and PMAX (for aggregation in 15min window conversion)
-    df_link['PMIN'] = df_link['REMOTERXLEVEL']
-    df_link['PMAX'] = df_link['REMOTERXLEVEL']
-    _log('Created PMIN and PMAX of REMOTERXLEVEL')
-    
-    # convert 3min windows to 15min windows
-    group_cols = [df_link['BEGINTIME'].dt.round('15Min'), 'RADIOLINKID']
+    # copy REMOTERXLEVEL to PMIN and PMAX (for aggregation in 15min time window)
+    df_link_window = df_link[['BEGINTIME', 'RADIOLINKID', 'REMOTERXLEVEL']].copy()
+    df_link_window['PMIN'] = df_link_window['REMOTERXLEVEL']
+    df_link_window['PMAX'] = df_link_window['REMOTERXLEVEL']
+    df_link_window['PMEAN'] = df_link_window['REMOTERXLEVEL']
+    df_link_window = df_link_window.drop('REMOTERXLEVEL', axis = 1)
+    _log('Created PMIN, PMAX and PMEAN of REMOTERXLEVEL')
+
+    # convert 3min windows to 15min windows to get min, max and mean power levels
+    group_cols = [df_link_window['BEGINTIME'].dt.round('15Min'), 'RADIOLINKID']
    agg_cols = {
        'PMIN' : 'min',
        'PMAX' : 'max',
-        'REMOTERXLEVEL' : 'mean',
-        'TXLEVEL' : 'mean',
-        'SPEED' : 'mean',
-        'CURRRXBITRATE' : 'mean',
-        'CURRTXBITRATE' : 'mean',
-        'CURRRXPROFILE' : 'median',
-        'CURRTXPROFILE' : 'median'
+        'PMEAN' : 'mean'
    }
-    df_link = df_link.groupby(group_cols).agg(agg_cols).reset_index()
-    _log('Converted 3min windows to 15min windows')
-    
-    # convert BEGINTIME to RAINLINK format
+    df_link_window = df_link_window.groupby(group_cols).agg(agg_cols).reset_index()
+    _log('Converted 3min windows to 15min windows to get min, max and mean power levels')
+    
+    # merge min, max and mean power levels for 15min windows back to link dataframe
+    # only get minutes 00, 15, 30 and 45 (PMIN, PMAX and PMEAN are not NaN --> faster than filtering datetime)
+    df_link = pd.merge(df_link, df_link_window, how = 'left', left_on = ['BEGINTIME', 'RADIOLINKID'], right_on = ['BEGINTIME', 'RADIOLINKID'])
+    del df_link_window
+    _log('Merged min, max and mean power levels for 15min windows back to link dataframe')
+    df_link = df_link.dropna(subset = ['PMIN', 'PMAX', 'PMEAN'])
+    _log('Use only transmissions in minutes 00, 15, 30 and 45')
+    
+    # convert BEGINTIME to format "%Y%m%d%H%M"
    df_link['BEGINTIME'] = df_link['BEGINTIME'].dt.strftime('%Y%m%d%H%M')
-    _log('Converted BEGINTIME to RAINLINK format "%Y%m%d%H%M"')
+    _log('Converted BEGINTIME to format "%Y%m%d%H%M"')
    
    
    # merge config and link dataframe
@@ -532,49 +534,45 @@ def prep() -> None:
    
    # build df with differences of sending and receiving levels
    df_diff = df_link[['LINKID', 'TXLEVEL', 'REMOTERXLEVEL']].copy()
-    df_diff['PINSTMEAN'] = df_diff['TXLEVEL'] - df_diff['REMOTERXLEVEL']
-    _log('Built dataframe with mean link difference levels of TXLEVEL and REMOTERXLEVEL')
+    df_diff['ATTENUATIONMEAN'] = df_diff['REMOTERXLEVEL'] - df_diff['TXLEVEL']
+    _log('Built dataframe with mean link attenuations')
    
    # get mean of differences per link
-    df_diff = df_diff.groupby(['LINKID']).agg({'PINSTMEAN' : 'mean'}).reset_index()
-    
+    df_diff = df_diff.groupby(['LINKID']).agg({'ATTENUATIONMEAN' : 'mean'}).reset_index()
    
    # merge differences to transmission dataframe
    df_link = pd.merge(df_link, df_diff, how = 'left', left_on = 'LINKID', right_on = 'LINKID')
-    _log('Merged mean link difference levels back to link dataframe')
-    
+    del df_diff
+    _log('Merged mean link attenuations back to link dataframe')
    
-    # calculate instantaneous values from transmitted and received signal strength
-    df_link['PINST'] = df_link['TXLEVEL'] - df_link['REMOTERXLEVEL']
-    _log('Calculated instantaneous values PINST from transmitted and received signal strength')
-    
-    # calculate differences from instantaneous values and mean link diff. levels
-    df_link['PDIFF'] = df_link['PINST'] - df_link['PINSTMEAN']
-    _log('Calculated difference levels PDIFF instantaneous values derivating of daily means of dry periods')
    
    
    # calculate attenuation of transmitted and received signal strengths per transmission
    df_link['ATTENUATION'] = df_link['REMOTERXLEVEL'] - df_link['TXLEVEL']
    _log('Calculated attenuation per transmission')
+
+    # calculate differences from attenuations and mean attenuations
+    df_link['ATTENUATIONDIFF'] = df_link['ATTENUATION'] - df_link['ATTENUATIONMEAN']
+    _log('Calculated attenuation differences of daily means')
    
    # calculate extinction coefficients for links based on attenuation and lengths of the links based on beer-lambert law
    # Coeff = -ln(10) * (Attenuation/PathLength)
-    df_link['EXTINCTIONCOEFF'] = -np.log(10) * (df_link['ATTENUATION'] / (df_link['LENGTH']/1000))
+    df_link['EXTINCTIONCOEFF'] = -np.log(10) * (df_link['ATTENUATION'] / (df_link['LENGTH']))
    _log('Calculated extinction coefficients based on beer-lambert law')
    
    
-    # rename and reorder columns to aid RAINLINK format
+    # rename and reorder columns
    name_cols = {
        'LINKID' : 'ID',
        'BEGINTIME' : 'DateTime',
-        'PMIN' : 'Pmin',
-        'PMAX' : 'Pmax',
-        'REMOTERXLEVEL' : 'Pmean',
-        'PINST' : 'Pinst',
-        'PINSTMEAN' : 'PinstMean',
-        'PDIFF' : 'Pdiff',
+        'REMOTERXLEVEL' : 'RxLevel',
+        'PMIN' : 'RxLevelMin',
+        'PMAX' : 'RxLevelMax',
+        'PMEAN' : 'RxLevelMean',
        'TXLEVEL' : 'TxLevel',
-        'ATTENUATION' : 'Attenuation',
+        'ATTENUATION' : 'Attn',
+        'ATTENUATIONMEAN' : 'AttnMean',
+        'ATTENUATIONDIFF' : 'AttnDiff',
        'EXTINCTIONCOEFF' : 'ExtinctionCoeff',
        'SPEED' : 'Speed',
        'CURRRXBITRATE' : 'RxBitrate',
@@ -602,7 +600,7 @@ def prep() -> None:
        })
    
    df_link = df_link.rename(columns = name_cols).reindex(columns = list(name_cols.values()))
-    _log('Converted link dataframe to RAINLINK format')
+    _log('Renamed and reordered link dataframe columns')
    
    
    _log('\n******************************** MERGE INCA ********************************')