Sort data directory.

.gitignore

@@ -10,6 +10,7 @@ __pycache__
 201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01.h5
 201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5
 Angu:arApp/prototype/node_modules
+backend/cache/
 clion-log.txt
 Documents/
 Flaskserver/venv/*

backend/src/main.py

@@ -13,7 +13,7 @@ from src import pseudo
 
 
 # Config
-path_preprocessed_data_npy = 'data/processed-data.npy'
+path_preprocessed_data_npy = 'cache/processed-data.npy'
 reload = False
 logging.basicConfig(level=logging.INFO)
 

backend/src/preprocessing.py

@@ -12,7 +12,7 @@ import tables
 from sklearn.preprocessing import minmax_scale
 
 logging.basicConfig(level=logging.INFO)
-data_path = "data/processed-data.npy"
+data_path = "cache/processed-data.npy"
 
 
 # def read_data():
@@ -51,7 +51,7 @@ def create_peax_windows_12kb_mts(window_size):
 
 def read_eeg_data(nr_of_channels):
     response = []
-    datafile = "data/21.csv"
+    datafile = "../data/21.csv"
     data = pd.read_csv(datafile, header=None)
     npdata = np.array(data, dtype="float32")
     del data
@@ -63,8 +63,8 @@ def read_eeg_data(nr_of_channels):
 
 
 def create_eeg_windows(window_size, nr_of_channels):
-    data_path = "data/processed-data_" + str(window_size) + ".npy"
-    datafile = "data/21.csv"
+    data_path = "../data/processed-data_" + str(window_size) + ".npy"
+    datafile = "../data/21.csv"
 
     if not os.path.isfile(data_path):
         data = pd.read_csv(datafile, header=None)
@@ -79,13 +79,13 @@ def create_eeg_windows(window_size, nr_of_channels):
         del npdata
         data = np.reshape(np_window_data, (len(np_window_data), nr_of_channels, len(np_window_data[0])))
         np.save(data_path, data)
-    np.save("data/processed-data", np.load(data_path))
+    np.save("../data/processed-data", np.load(data_path))
     return "1"
 
 
 def read_eeg_data(nr_of_channels):
     response = []
-    datafile = "data/21.csv"
+    datafile = "../data/21.csv"
     data = pd.read_csv(datafile, header=None)
     npdata = np.array(data, dtype="float32")
     del data
@@ -100,7 +100,7 @@ def read_weather_data():
     filename = "../data/weather.pkl"
     if not os.path.isfile(filename):
         print("start")
-        df = dd.read_csv("data/NW_Ground_Stations_2016.csv", usecols=["number_sta", "date", "t", "hu", "td", "dd", "ff", "psl", "precip"])
+        df = dd.read_csv("../data/NW_Ground_Stations_2016.csv", usecols=["number_sta", "date", "t", "hu", "td", "dd", "ff", "psl", "precip"])
         print("read file")
         df = df.loc[df["number_sta"].isin([14066001, 14137001, 14216001, 14372001, 22092001, 22113006, 22135001])].fillna(0)
         df["date"] = dd.to_datetime(df["date"], format="%Y%m%d %H:%M")
@@ -151,7 +151,7 @@ def read_weather_data():
 
 
 def create_weather_windows(window_size):
-    if not os.path.isfile("data/weather-" + str(window_size) + ".npy"):
+    if not os.path.isfile("cache/weather-" + str(window_size) + ".npy"):
         filename = "../data/weather.pkl"
         df = pd.read_pickle(filename)
         channels = list()
@@ -170,9 +170,9 @@ def create_weather_windows(window_size):
             windows.append(minmax_scale(data[i], (-1, 1), axis=1))
         print("dims:")
         print(windows[0].size)
-        np.save("data/weather-" + str(window_size), windows)
-    data = np.load("data/weather-" + str(window_size) + ".npy")
-    np.save("data/processed-data", data)
+        np.save("../data/weather-" + str(window_size), windows)
+    data = np.load("../data/weather-" + str(window_size) + ".npy")
+    np.save("../data/processed-data", data)
     return "1"
 
 
@@ -181,7 +181,7 @@ def read_egr_data():
     # Config
     path_data_original: Union[
         Path, str
-    ] = "data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
+    ] = "../data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
     channel_names = ["time", "ACM_Egrrate_demand_managed", "ACM_Egrrate_feedback_filt", "ACM_Egr_enable"]
 
     # Load data
@@ -203,9 +203,9 @@ def read_egr_data():
 def create_egr_windows(window_size):
     """Create windows for EGR dataset."""
     # Config
-    path_data_original_hdf = "data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
-    path_data_cached_npy = f"data/egr_cached_{window_size}.npy"
-    path_data_preprocessed_npy = f"data/processed-data.npy"
+    path_data_original_hdf = "../data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
+    path_data_cached_npy = f"cache/egr_cached_{window_size}.npy"
+    path_data_preprocessed_npy = f"cache/processed-data.npy"
 
     # Created cached data
     if not Path(path_data_cached_npy).is_file():

backend/src/pseudo.py

@@ -16,10 +16,10 @@ def get_lsh_parameters(data, window_size):
     """
     data: 3d array [m][t][d]
     """
-    if (not os.path.isfile('data/parameters-' + str(window_size) + '.npy')):
+    if (not os.path.isfile('cache/parameters-' + str(window_size) + '.npy')):
         parameters = preprocess(data)
-        np.save('data/parameters-' + str(window_size), [float(parameters[0]), float(parameters[1]), float(parameters[2])])
-    return np.load('data/parameters-' + str(window_size) + '.npy').tolist()
+        np.save('cache/parameters-' + str(window_size), [float(parameters[0]), float(parameters[1]), float(parameters[2])])
+    return np.load('cache/parameters-' + str(window_size) + '.npy').tolist()
 
 
 def lsh(data, query, parameters=None, weights=None):
@@ -269,7 +269,7 @@ def query(data, window_indices):
 
 
 def debug_test_lsh():
-    data = np.load('../data/processed-data.npy')
+    data = np.load('cache/processed-data.npy')
     # data = np.repeat(data, repeats=7, axis=1)
     print(data.shape)
     data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))