Skip to content
GitLab
Commit 8e3f5782 authored by Fabian Kovac's avatar Fabian Kovac
Browse files

[f] attenuation and entities of 15min-steps only

parent a03d3467
Hide whitespace changes
Inline Side-by-side
FHSTP/01_data_preparation/prep.py
View file @ 8e3f5782
......@@ -2,8 +2,8 @@
# Title: Data Preparation for LINK Configs and Transmissions
# Author: Fabian Kovac <ds191008@fhstp.ac.at>
# Team: University of Applied Sciences St. Pölten
# Version: 2.9
# Last changed: 2021-10-06
# Version: 3.0
# Last changed: 2021-10-22
#
import sys
......@@ -142,7 +142,7 @@ def get_midpoint(lon_a: np.array, lat_a: np.array, lon_b: np.array, lat_b: np.ar
def get_distance(lon_a: np.array, lat_a: np.array, alt_a: np.array, lon_b: np.array, lat_b: np.array, alt_b: np.array) -> np.array:
"""Calculcates distance between two coordinates in m using a WGS84 rotation-ellipsoid
"""Calculcates distance between two coordinates in km using a WGS84 rotation-ellipsoid
Parameters:
lon_a (np.array): Longitudes of point A
......@@ -153,13 +153,13 @@ def get_distance(lon_a: np.array, lat_a: np.array, alt_a: np.array, lon_b: np.ar
alt_b (np.array): Altitudes of point B
Returns:
length (np.array): Vector with distances in m
length (np.array): Vector with distances in km
"""
# constants (equator radius and pole radius in m)
# constants (equator radius and pole radius in km)
# r_equator is the 'semi-major axis' and r_pole the 'semi-minor axis' on a WGS84 ellipsoid
r_equator = 6378137
r_pole = 6356752.3142
r_equator = 6378.137
r_pole = 6356.7523142
# calculate points on rotation-ellipsoid
# point A
......@@ -307,7 +307,7 @@ def get_inca_data(datetimes: np.array, lon_a: np.array, lat_a: np.array, lon_b:
# define n INCA RR points for each km of link length
# --> min. points: 3 (start, mid and end) if length < 3km
# --> get odd amount of RR points to always include the middle of the link
n = np.round(lengths/1000, decimals = 0).astype(int)
n = np.round(lengths, decimals = 0).astype(int)
n = np.where(n < 3, 3, n)
n = np.where(n%2 == 0, n+1, n)
......@@ -426,6 +426,7 @@ def prep() -> None:
df_altitudes = pd.read_csv('files_from_H3A/CONFIG_ALTITUDES.csv', sep = ';')
df_altitudes = df_altitudes[['LINKID', 'ANT_HEIGHT_ALTITUDE_A', 'ANT_HEIGHT_ALTITUDE_B']]
df_config = pd.merge(df_config, df_altitudes, how = 'inner', left_on = 'LINKID', right_on = 'LINKID')
del df_altitudes
_log('Added altitude of links')
......@@ -438,7 +439,7 @@ def prep() -> None:
df_config['ANT_HEIGHT_ALTITUDE_MID'] = df_config['ANT_HEIGHT_ALTITUDE_MID'].astype('int')
_log('Calculated midpoint of links')
# calculate LENGTH in m between links
# calculate LENGTH in km between links
df_config['LENGTH'] = get_distance(
df_config['LONGITUDE_A'],
df_config['LATITUDE_A'],
......@@ -486,40 +487,41 @@ def prep() -> None:
df_link['BEGINTIME'] = df_link['BEGINTIME'].dt.tz_localize('Europe/Vienna').dt.tz_convert('UTC').dt.tz_localize(None)
_log('Converted BEGINTIME to UTC')
# fix 15min timelag in transmissions
df_link['BEGINTIME'] = df_link['BEGINTIME'] - pd.Timedelta(15, unit = 'min')
_log('Fixed 15min timelag in transmissions')
# only use transmissions with begintime matching INCA date
date_inca = np.datetime64(f'{str(dir_inca.stem)[0:4]}-{str(dir_inca.stem)[-4:-2]}-{str(dir_inca.stem)[-2:]}')
df_link = df_link[df_link['BEGINTIME'].dt.date == date_inca]
_log('Filtered transmissions to match BEGINTIME with INCA date')
# copy REMOTERXLEVEL to PMIN and PMAX (for aggregation in 15min window conversion)
df_link['PMIN'] = df_link['REMOTERXLEVEL']
df_link['PMAX'] = df_link['REMOTERXLEVEL']
_log('Created PMIN and PMAX of REMOTERXLEVEL')
# convert 3min windows to 15min windows
group_cols = [df_link['BEGINTIME'].dt.round('15Min'), 'RADIOLINKID']
# copy REMOTERXLEVEL to PMIN and PMAX (for aggregation in 15min time window)
df_link_window = df_link[['BEGINTIME', 'RADIOLINKID', 'REMOTERXLEVEL']].copy()
df_link_window['PMIN'] = df_link_window['REMOTERXLEVEL']
df_link_window['PMAX'] = df_link_window['REMOTERXLEVEL']
df_link_window['PMEAN'] = df_link_window['REMOTERXLEVEL']
df_link_window = df_link_window.drop('REMOTERXLEVEL', axis = 1)
_log('Created PMIN, PMAX and PMEAN of REMOTERXLEVEL')
# convert 3min windows to 15min windows to get min, max and mean power levels
group_cols = [df_link_window['BEGINTIME'].dt.round('15Min'), 'RADIOLINKID']
agg_cols = {
'PMIN' : 'min',
'PMAX' : 'max',
'REMOTERXLEVEL' : 'mean',
'TXLEVEL' : 'mean',
'SPEED' : 'mean',
'CURRRXBITRATE' : 'mean',
'CURRTXBITRATE' : 'mean',
'CURRRXPROFILE' : 'median',
'CURRTXPROFILE' : 'median'
'PMEAN' : 'mean'
}
df_link = df_link.groupby(group_cols).agg(agg_cols).reset_index()
_log('Converted 3min windows to 15min windows')
# convert BEGINTIME to RAINLINK format
df_link_window = df_link_window.groupby(group_cols).agg(agg_cols).reset_index()
_log('Converted 3min windows to 15min windows to get min, max and mean power levels')
# merge min, max and mean power levels for 15min windows back to link dataframe
# only get minutes 00, 15, 30 and 45 (PMIN, PMAX and PMEAN are not NaN --> faster than filtering datetime)
df_link = pd.merge(df_link, df_link_window, how = 'left', left_on = ['BEGINTIME', 'RADIOLINKID'], right_on = ['BEGINTIME', 'RADIOLINKID'])
del df_link_window
_log('Merged min, max and mean power levels for 15min windows back to link dataframe')
df_link = df_link.dropna(subset = ['PMIN', 'PMAX', 'PMEAN'])
_log('Use only transmissions in minutes 00, 15, 30 and 45')
# convert BEGINTIME to format "%Y%m%d%H%M"
df_link['BEGINTIME'] = df_link['BEGINTIME'].dt.strftime('%Y%m%d%H%M')
_log('Converted BEGINTIME to RAINLINK format "%Y%m%d%H%M"')
_log('Converted BEGINTIME to format "%Y%m%d%H%M"')
# merge config and link dataframe
......@@ -532,49 +534,45 @@ def prep() -> None:
# build df with differences of sending and receiving levels
df_diff = df_link[['LINKID', 'TXLEVEL', 'REMOTERXLEVEL']].copy()
df_diff['PINSTMEAN'] = df_diff['TXLEVEL'] - df_diff['REMOTERXLEVEL']
_log('Built dataframe with mean link difference levels of TXLEVEL and REMOTERXLEVEL')
df_diff['ATTENUATIONMEAN'] = df_diff['REMOTERXLEVEL'] - df_diff['TXLEVEL']
_log('Built dataframe with mean link attenuations')
# get mean of differences per link
df_diff = df_diff.groupby(['LINKID']).agg({'PINSTMEAN' : 'mean'}).reset_index()
df_diff = df_diff.groupby(['LINKID']).agg({'ATTENUATIONMEAN' : 'mean'}).reset_index()
# merge differences to transmission dataframe
df_link = pd.merge(df_link, df_diff, how = 'left', left_on = 'LINKID', right_on = 'LINKID')
_log('Merged mean link difference levels back to link dataframe')
del df_diff
_log('Merged mean link attenuations back to link dataframe')
# calculate instantaneous values from transmitted and received signal strength
df_link['PINST'] = df_link['TXLEVEL'] - df_link['REMOTERXLEVEL']
_log('Calculated instantaneous values PINST from transmitted and received signal strength')
# calculate differences from instantaneous values and mean link diff. levels
df_link['PDIFF'] = df_link['PINST'] - df_link['PINSTMEAN']
_log('Calculated difference levels PDIFF instantaneous values derivating of daily means of dry periods')
# calculate attenuation of transmitted and received signal strengths per transmission
df_link['ATTENUATION'] = df_link['REMOTERXLEVEL'] - df_link['TXLEVEL']
_log('Calculated attenuation per transmission')
# calculate differences from attenuations and mean attenuations
df_link['ATTENUATIONDIFF'] = df_link['ATTENUATION'] - df_link['ATTENUATIONMEAN']
_log('Calculated attenuation differences of daily means')
# calculate extinction coefficients for links based on attenuation and lengths of the links based on beer-lambert law
# Coeff = -ln(10) * (Attenuation/PathLength)
df_link['EXTINCTIONCOEFF'] = -np.log(10) * (df_link['ATTENUATION'] / (df_link['LENGTH']/1000))
df_link['EXTINCTIONCOEFF'] = -np.log(10) * (df_link['ATTENUATION'] / (df_link['LENGTH']))
_log('Calculated extinction coefficients based on beer-lambert law')
# rename and reorder columns to aid RAINLINK format
# rename and reorder columns
name_cols = {
'LINKID' : 'ID',
'BEGINTIME' : 'DateTime',
'PMIN' : 'Pmin',
'PMAX' : 'Pmax',
'REMOTERXLEVEL' : 'Pmean',
'PINST' : 'Pinst',
'PINSTMEAN' : 'PinstMean',
'PDIFF' : 'Pdiff',
'REMOTERXLEVEL' : 'RxLevel',
'PMIN' : 'RxLevelMin',
'PMAX' : 'RxLevelMax',
'PMEAN' : 'RxLevelMean',
'TXLEVEL' : 'TxLevel',
'ATTENUATION' : 'Attenuation',
'ATTENUATION' : 'Attn',
'ATTENUATIONMEAN' : 'AttnMean',
'ATTENUATIONDIFF' : 'AttnDiff',
'EXTINCTIONCOEFF' : 'ExtinctionCoeff',
'SPEED' : 'Speed',
'CURRRXBITRATE' : 'RxBitrate',
......@@ -602,7 +600,7 @@ def prep() -> None:
})
df_link = df_link.rename(columns = name_cols).reindex(columns = list(name_cols.values()))
_log('Converted link dataframe to RAINLINK format')
_log('Renamed and reordered link dataframe columns')
_log('\n******************************** MERGE INCA ********************************')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment