Commit 33ced1b8 authored by Yuncong Yu's avatar Yuncong Yu
Browse files

Unify input APIs; add EEG eye state dataset.

parent be2e7008
import logging import logging
import yaml
from pathlib import Path from pathlib import Path
from time import time, perf_counter from time import time, perf_counter
from typing import Any, Dict, List, Union from typing import Any, Dict, List, Union
import numpy as np import numpy as np
import orjson import orjson
import yaml
from flask import Flask, request from flask import Flask, request
from flask_cors import CORS from flask_cors import CORS
from src import preprocessing
from src import pseudo from src import pseudo
from src.preprocessing import Preprocessor
logging.basicConfig(level=logging.INFO)
# Read config # Read config
...@@ -27,13 +29,14 @@ def read_config(path_global_config_yml: Union[Path, str] = Path("../config.yml") ...@@ -27,13 +29,14 @@ def read_config(path_global_config_yml: Union[Path, str] = Path("../config.yml")
config = read_config() config = read_config()
# path_preprocessed_data_npy = 'cache/preprocessed-data.npy' preprossessor = Preprocessor(
path_data_hdf = Path(config["path_data_hdf"]) dataset=config["dataset"],
path_meta_json = Path(config["path_meta_json"]) path_data_hdf=Path(config["path_data_hdf"]),
path_preprocessed_data_npy = Path(config["dir_cache"]) / "preprocessed_data.npy" path_meta_json=Path(config["path_meta_json"]),
channel_names = config["channels"] channel_names=config["channels"],
dir_in_hdf = config["dir_in_hdf"] dir_in_hdf=config["dir_in_hdf"],
logging.basicConfig(level=logging.INFO) dir_cache=Path(config["dir_cache"]),
)
app = Flask(__name__) app = Flask(__name__)
CORS(app) CORS(app)
...@@ -64,11 +67,7 @@ def read_data(): ...@@ -64,11 +67,7 @@ def read_data():
logging.info("Loading data ...") logging.info("Loading data ...")
time_start = perf_counter() time_start = perf_counter()
# response = preprocessing.read_weather_data() response = preprossessor.create_response()
# response = preprocessing.read_egr_data()
response = preprocessing.read_data(
path_data_hdf=path_data_hdf, path_meta_json=path_meta_json, channel_names=channel_names, dir_in_hdf=dir_in_hdf
)
response = orjson.dumps(response) response = orjson.dumps(response)
logging.info(f"Completed loading data with {perf_counter() - time_start:.2f} second(s).") logging.info(f"Completed loading data with {perf_counter() - time_start:.2f} second(s).")
...@@ -97,11 +96,10 @@ def create_windows(): ...@@ -97,11 +96,10 @@ def create_windows():
logging.info("Creating window ...") logging.info("Creating window ...")
time_start = perf_counter() time_start = perf_counter()
if not Path(path_preprocessed_data_npy).is_file(): if not preprossessor.path_preprocessed_data_npy_.is_file():
raw_data = request.json raw_data = request.json
window_size = int(raw_data["parameters"]["windowsize"]) window_size = int(raw_data["parameters"]["windowsize"])
# preprocessing.create_eeg_windows(window_size, 5) preprossessor.create_windows(window_size)
preprocessing.create_egr_windows(window_size)
logging.info(f"Completed windows with {perf_counter() - time_start:.2f} second(s).") logging.info(f"Completed windows with {perf_counter() - time_start:.2f} second(s).")
...@@ -145,7 +143,7 @@ def initialize(): ...@@ -145,7 +143,7 @@ def initialize():
# Read windows # Read windows
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
data_windowized = np.load(path_preprocessed_data_npy) data_windowized = np.load(preprossessor.path_preprocessed_data_npy_)
data_windowized = np.swapaxes(data_windowized, 1, 2) # Use a column vector for each channel data_windowized = np.swapaxes(data_windowized, 1, 2) # Use a column vector for each channel
# Read the query # Read the query
...@@ -179,7 +177,7 @@ def get_lsh_parameters(): ...@@ -179,7 +177,7 @@ def get_lsh_parameters():
t0 = time() t0 = time()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
window_size = raw_data["windowsize"] window_size = raw_data["windowsize"]
data = np.load(path_preprocessed_data_npy) data = np.load(preprossessor.path_preprocessed_data_npy_)
data = np.swapaxes(data, 1, 2) data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data, window_size) parameters = pseudo.get_lsh_parameters(data, window_size)
...@@ -214,7 +212,7 @@ def update(): ...@@ -214,7 +212,7 @@ def update():
""" """
t0 = time() t0 = time()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
data = np.load(path_preprocessed_data_npy) data = np.load(preprossessor.path_preprocessed_data_npy_)
data = np.swapaxes(data, 1, 2) data = np.swapaxes(data, 1, 2)
query = raw_data["query"] query = raw_data["query"]
query = np.swapaxes(query, 0, 1) query = np.swapaxes(query, 0, 1)
...@@ -247,7 +245,7 @@ def weights(): ...@@ -247,7 +245,7 @@ def weights():
hash_functions = raw_data["hash_functions"] hash_functions = raw_data["hash_functions"]
query = raw_data["query"] query = raw_data["query"]
old_weights = raw_data["weights"] old_weights = raw_data["weights"]
data = np.load(path_preprocessed_data_npy) data = np.load(preprossessor.path_preprocessed_data_npy_)
new_weights = pseudo.weights(data, query, old_weights, labels, hash_functions) new_weights = pseudo.weights(data, query, old_weights, labels, hash_functions)
...@@ -271,17 +269,15 @@ def query(): ...@@ -271,17 +269,15 @@ def query():
time_start = perf_counter() time_start = perf_counter()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
# print(raw_data)
start_index = raw_data["start_index"] start_index = raw_data["start_index"]
query_size = raw_data["query_size"] query_size = raw_data["query_size"]
window_indices = raw_data["indices"] window_indices = raw_data["indices"]
if start_index is not None: if start_index is not None:
# preprocessing.create_weather_windows(query_size) preprossessor.create_windows(query_size)
preprocessing.create_egr_windows(query_size)
window_indices = int(start_index) window_indices = int(start_index)
data_windowized = np.load(path_preprocessed_data_npy) data_windowized = np.load(preprossessor.path_preprocessed_data_npy_)
response = pseudo.query(data_windowized, window_indices) response = pseudo.query(data_windowized, window_indices)
response = orjson.dumps(response) response = orjson.dumps(response)
...@@ -305,7 +301,7 @@ def window(): ...@@ -305,7 +301,7 @@ def window():
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
indices = raw_data["indices"] indices = raw_data["indices"]
output = np.load(path_preprocessed_data_npy)[indices] output = np.load(preprossessor.path_preprocessed_data_npy_)[indices]
response = orjson.dumps(output.tolist()) response = orjson.dumps(output.tolist())
print("Window(s) done: " + str(time() - t0)) print("Window(s) done: " + str(time() - t0))
...@@ -333,7 +329,7 @@ def table_info(): ...@@ -333,7 +329,7 @@ def table_info():
t0 = time() t0 = time()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
table = raw_data["table"] table = raw_data["table"]
data = np.load(path_preprocessed_data_npy) data = np.load(preprossessor.path_preprocessed_data_npy_)
response = pseudo.table_info(data, table) response = pseudo.table_info(data, table)
......
This diff is collapsed.
...@@ -3,4 +3,5 @@ ...@@ -3,4 +3,5 @@
reload: false reload: false
# Specific configuration file # Specific configuration file
path_config_yml: D:\Projects\pseudo\experiments\configs\config_egr.yml path_config_yml: D:\Projects\pseudo\experiments\configs\config_egr.yml
\ No newline at end of file #path_config_yml: D:\Projects\pseudo\experiments\configs\config_eeg_eye_state.yml
\ No newline at end of file
[
{
"time_start": 1.4684558381734427,
"time_end": 6.803324654516323,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 10.435409573402763,
"time_end": 12.794312036851592,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 16.99659523332666,
"time_end": 20.56619266973763,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 22.6517123973563,
"time_end": 22.862607650710995,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 26.104145804125775,
"time_end": 33.99319046665332,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 40.96054476266773,
"time_end": 46.30322451432005,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 51.96615261365912,
"time_end": 70.72020829160824,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 86.74043661125576,
"time_end": 94.3248547967154,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 99.41758461846585,
"time_end": 99.75345483677148,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 101.35469657520528,
"time_end": 101.76086521129581,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 111.04806729421189,
"time_end": 111.61045463649108,
"file": "eeg_eye_state.hdf"
}
]
\ No newline at end of file
{
"short_names": [
"time",
"AF3",
"F7",
"F3",
"FC5",
"T7",
"P7",
"O1",
"O2",
"P8",
"T8",
"FC6",
"F4",
"F8",
"AF4"
],
"long_names": [
"time",
"AF3",
"F7",
"F3",
"FC5",
"T7",
"P7",
"O1",
"O2",
"P8",
"T8",
"FC6",
"F4",
"F8",
"AF4"
],
"units": [
"s",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-"
]
}
\ No newline at end of file
---
dataset: eeg_eye_state
path_data_hdf: D:\Projects\pseudo\data\eeg_eye_state\eeg_eye_state.hdf
path_meta_json: D:\Projects\pseudo\data\eeg_eye_state\metadata.json
channels:
- time
- AF3
- F7
- F3
- FC5
- T7
- P7
- O1
- O2
- P8
- T8
- FC6
- F4
- F8
- AF4
dir_in_hdf: resampled
dir_cache: cache\
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment