Commit 6ef8f879 authored by Yuncong Yu's avatar Yuncong Yu
Browse files

Sort data directory.

parent bfd51f4c
...@@ -10,6 +10,7 @@ __pycache__ ...@@ -10,6 +10,7 @@ __pycache__
201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01.h5 201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01.h5
201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5 201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5
Angu:arApp/prototype/node_modules Angu:arApp/prototype/node_modules
backend/cache/
clion-log.txt clion-log.txt
Documents/ Documents/
Flaskserver/venv/* Flaskserver/venv/*
......
...@@ -13,7 +13,7 @@ from src import pseudo ...@@ -13,7 +13,7 @@ from src import pseudo
# Config # Config
path_preprocessed_data_npy = 'data/processed-data.npy' path_preprocessed_data_npy = 'cache/processed-data.npy'
reload = False reload = False
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
......
...@@ -12,7 +12,7 @@ import tables ...@@ -12,7 +12,7 @@ import tables
from sklearn.preprocessing import minmax_scale from sklearn.preprocessing import minmax_scale
logging.basicConfig(level=logging.INFO) logging.basicConfig(level=logging.INFO)
data_path = "data/processed-data.npy" data_path = "cache/processed-data.npy"
# def read_data(): # def read_data():
...@@ -51,7 +51,7 @@ def create_peax_windows_12kb_mts(window_size): ...@@ -51,7 +51,7 @@ def create_peax_windows_12kb_mts(window_size):
def read_eeg_data(nr_of_channels): def read_eeg_data(nr_of_channels):
response = [] response = []
datafile = "data/21.csv" datafile = "../data/21.csv"
data = pd.read_csv(datafile, header=None) data = pd.read_csv(datafile, header=None)
npdata = np.array(data, dtype="float32") npdata = np.array(data, dtype="float32")
del data del data
...@@ -63,8 +63,8 @@ def read_eeg_data(nr_of_channels): ...@@ -63,8 +63,8 @@ def read_eeg_data(nr_of_channels):
def create_eeg_windows(window_size, nr_of_channels): def create_eeg_windows(window_size, nr_of_channels):
data_path = "data/processed-data_" + str(window_size) + ".npy" data_path = "../data/processed-data_" + str(window_size) + ".npy"
datafile = "data/21.csv" datafile = "../data/21.csv"
if not os.path.isfile(data_path): if not os.path.isfile(data_path):
data = pd.read_csv(datafile, header=None) data = pd.read_csv(datafile, header=None)
...@@ -79,13 +79,13 @@ def create_eeg_windows(window_size, nr_of_channels): ...@@ -79,13 +79,13 @@ def create_eeg_windows(window_size, nr_of_channels):
del npdata del npdata
data = np.reshape(np_window_data, (len(np_window_data), nr_of_channels, len(np_window_data[0]))) data = np.reshape(np_window_data, (len(np_window_data), nr_of_channels, len(np_window_data[0])))
np.save(data_path, data) np.save(data_path, data)
np.save("data/processed-data", np.load(data_path)) np.save("../data/processed-data", np.load(data_path))
return "1" return "1"
def read_eeg_data(nr_of_channels): def read_eeg_data(nr_of_channels):
response = [] response = []
datafile = "data/21.csv" datafile = "../data/21.csv"
data = pd.read_csv(datafile, header=None) data = pd.read_csv(datafile, header=None)
npdata = np.array(data, dtype="float32") npdata = np.array(data, dtype="float32")
del data del data
...@@ -100,7 +100,7 @@ def read_weather_data(): ...@@ -100,7 +100,7 @@ def read_weather_data():
filename = "../data/weather.pkl" filename = "../data/weather.pkl"
if not os.path.isfile(filename): if not os.path.isfile(filename):
print("start") print("start")
df = dd.read_csv("data/NW_Ground_Stations_2016.csv", usecols=["number_sta", "date", "t", "hu", "td", "dd", "ff", "psl", "precip"]) df = dd.read_csv("../data/NW_Ground_Stations_2016.csv", usecols=["number_sta", "date", "t", "hu", "td", "dd", "ff", "psl", "precip"])
print("read file") print("read file")
df = df.loc[df["number_sta"].isin([14066001, 14137001, 14216001, 14372001, 22092001, 22113006, 22135001])].fillna(0) df = df.loc[df["number_sta"].isin([14066001, 14137001, 14216001, 14372001, 22092001, 22113006, 22135001])].fillna(0)
df["date"] = dd.to_datetime(df["date"], format="%Y%m%d %H:%M") df["date"] = dd.to_datetime(df["date"], format="%Y%m%d %H:%M")
...@@ -151,7 +151,7 @@ def read_weather_data(): ...@@ -151,7 +151,7 @@ def read_weather_data():
def create_weather_windows(window_size): def create_weather_windows(window_size):
if not os.path.isfile("data/weather-" + str(window_size) + ".npy"): if not os.path.isfile("cache/weather-" + str(window_size) + ".npy"):
filename = "../data/weather.pkl" filename = "../data/weather.pkl"
df = pd.read_pickle(filename) df = pd.read_pickle(filename)
channels = list() channels = list()
...@@ -170,9 +170,9 @@ def create_weather_windows(window_size): ...@@ -170,9 +170,9 @@ def create_weather_windows(window_size):
windows.append(minmax_scale(data[i], (-1, 1), axis=1)) windows.append(minmax_scale(data[i], (-1, 1), axis=1))
print("dims:") print("dims:")
print(windows[0].size) print(windows[0].size)
np.save("data/weather-" + str(window_size), windows) np.save("../data/weather-" + str(window_size), windows)
data = np.load("data/weather-" + str(window_size) + ".npy") data = np.load("../data/weather-" + str(window_size) + ".npy")
np.save("data/processed-data", data) np.save("../data/processed-data", data)
return "1" return "1"
...@@ -181,7 +181,7 @@ def read_egr_data(): ...@@ -181,7 +181,7 @@ def read_egr_data():
# Config # Config
path_data_original: Union[ path_data_original: Union[
Path, str Path, str
] = "data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5" ] = "../data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
channel_names = ["time", "ACM_Egrrate_demand_managed", "ACM_Egrrate_feedback_filt", "ACM_Egr_enable"] channel_names = ["time", "ACM_Egrrate_demand_managed", "ACM_Egrrate_feedback_filt", "ACM_Egr_enable"]
# Load data # Load data
...@@ -203,9 +203,9 @@ def read_egr_data(): ...@@ -203,9 +203,9 @@ def read_egr_data():
def create_egr_windows(window_size): def create_egr_windows(window_size):
"""Create windows for EGR dataset.""" """Create windows for EGR dataset."""
# Config # Config
path_data_original_hdf = "data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5" path_data_original_hdf = "../data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
path_data_cached_npy = f"data/egr_cached_{window_size}.npy" path_data_cached_npy = f"cache/egr_cached_{window_size}.npy"
path_data_preprocessed_npy = f"data/processed-data.npy" path_data_preprocessed_npy = f"cache/processed-data.npy"
# Created cached data # Created cached data
if not Path(path_data_cached_npy).is_file(): if not Path(path_data_cached_npy).is_file():
......
...@@ -16,10 +16,10 @@ def get_lsh_parameters(data, window_size): ...@@ -16,10 +16,10 @@ def get_lsh_parameters(data, window_size):
""" """
data: 3d array [m][t][d] data: 3d array [m][t][d]
""" """
if (not os.path.isfile('data/parameters-' + str(window_size) + '.npy')): if (not os.path.isfile('cache/parameters-' + str(window_size) + '.npy')):
parameters = preprocess(data) parameters = preprocess(data)
np.save('data/parameters-' + str(window_size), [float(parameters[0]), float(parameters[1]), float(parameters[2])]) np.save('cache/parameters-' + str(window_size), [float(parameters[0]), float(parameters[1]), float(parameters[2])])
return np.load('data/parameters-' + str(window_size) + '.npy').tolist() return np.load('cache/parameters-' + str(window_size) + '.npy').tolist()
def lsh(data, query, parameters=None, weights=None): def lsh(data, query, parameters=None, weights=None):
...@@ -269,7 +269,7 @@ def query(data, window_indices): ...@@ -269,7 +269,7 @@ def query(data, window_indices):
def debug_test_lsh(): def debug_test_lsh():
data = np.load('../data/processed-data.npy') data = np.load('cache/processed-data.npy')
# data = np.repeat(data, repeats=7, axis=1) # data = np.repeat(data, repeats=7, axis=1)
print(data.shape) print(data.shape)
data = np.reshape(data, (len(data), len(data[0][0]), len(data[0]))) data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment