Skip to content

GitLab

Commit 33ced1b8 authored by Yuncong Yu's avatar Yuncong Yu
Browse files

Unify input APIs; add EEG eye state dataset.

parent be2e7008
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 508 additions and 265 deletions
backend/src/main.py
View file @ 33ced1b8
import logging
import yaml
from pathlib import Path
from time import time, perf_counter
from typing import Any, Dict, List, Union
import numpy as np
import orjson
import yaml
from flask import Flask, request
from flask_cors import CORS
from src import preprocessing
from src import pseudo
from src.preprocessing import Preprocessor
logging.basicConfig(level=logging.INFO)
# Read config
......@@ -27,13 +29,14 @@ def read_config(path_global_config_yml: Union[Path, str] = Path("../config.yml")
config = read_config()
# path_preprocessed_data_npy = 'cache/preprocessed-data.npy'
path_data_hdf = Path(config["path_data_hdf"])
path_meta_json = Path(config["path_meta_json"])
path_preprocessed_data_npy = Path(config["dir_cache"]) / "preprocessed_data.npy"
channel_names = config["channels"]
dir_in_hdf = config["dir_in_hdf"]
logging.basicConfig(level=logging.INFO)
preprossessor = Preprocessor(
dataset=config["dataset"],
path_data_hdf=Path(config["path_data_hdf"]),
path_meta_json=Path(config["path_meta_json"]),
channel_names=config["channels"],
dir_in_hdf=config["dir_in_hdf"],
dir_cache=Path(config["dir_cache"]),
)
app = Flask(__name__)
CORS(app)
......@@ -64,11 +67,7 @@ def read_data():
logging.info("Loading data ...")
time_start = perf_counter()
# response = preprocessing.read_weather_data()
# response = preprocessing.read_egr_data()
response = preprocessing.read_data(
path_data_hdf=path_data_hdf, path_meta_json=path_meta_json, channel_names=channel_names, dir_in_hdf=dir_in_hdf
)
response = preprossessor.create_response()
response = orjson.dumps(response)
logging.info(f"Completed loading data with {perf_counter() - time_start:.2f} second(s).")
......@@ -97,11 +96,10 @@ def create_windows():
logging.info("Creating window ...")
time_start = perf_counter()
if not Path(path_preprocessed_data_npy).is_file():
if not preprossessor.path_preprocessed_data_npy_.is_file():
raw_data = request.json
window_size = int(raw_data["parameters"]["windowsize"])
# preprocessing.create_eeg_windows(window_size, 5)
preprocessing.create_egr_windows(window_size)
preprossessor.create_windows(window_size)
logging.info(f"Completed windows with {perf_counter() - time_start:.2f} second(s).")
......@@ -145,7 +143,7 @@ def initialize():
# Read windows
raw_data = orjson.loads(request.data)
data_windowized = np.load(path_preprocessed_data_npy)
data_windowized = np.load(preprossessor.path_preprocessed_data_npy_)
data_windowized = np.swapaxes(data_windowized, 1, 2) # Use a column vector for each channel
# Read the query
......@@ -179,7 +177,7 @@ def get_lsh_parameters():
t0 = time()
raw_data = orjson.loads(request.data)
window_size = raw_data["windowsize"]
data = np.load(path_preprocessed_data_npy)
data = np.load(preprossessor.path_preprocessed_data_npy_)
data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data, window_size)
......@@ -214,7 +212,7 @@ def update():
"""
t0 = time()
raw_data = orjson.loads(request.data)
data = np.load(path_preprocessed_data_npy)
data = np.load(preprossessor.path_preprocessed_data_npy_)
data = np.swapaxes(data, 1, 2)
query = raw_data["query"]
query = np.swapaxes(query, 0, 1)
......@@ -247,7 +245,7 @@ def weights():
hash_functions = raw_data["hash_functions"]
query = raw_data["query"]
old_weights = raw_data["weights"]
data = np.load(path_preprocessed_data_npy)
data = np.load(preprossessor.path_preprocessed_data_npy_)
new_weights = pseudo.weights(data, query, old_weights, labels, hash_functions)
......@@ -271,17 +269,15 @@ def query():
time_start = perf_counter()
raw_data = orjson.loads(request.data)
# print(raw_data)
start_index = raw_data["start_index"]
query_size = raw_data["query_size"]
window_indices = raw_data["indices"]
if start_index is not None:
# preprocessing.create_weather_windows(query_size)
preprocessing.create_egr_windows(query_size)
preprossessor.create_windows(query_size)
window_indices = int(start_index)
data_windowized = np.load(path_preprocessed_data_npy)
data_windowized = np.load(preprossessor.path_preprocessed_data_npy_)
response = pseudo.query(data_windowized, window_indices)
response = orjson.dumps(response)
......@@ -305,7 +301,7 @@ def window():
raw_data = orjson.loads(request.data)
indices = raw_data["indices"]
output = np.load(path_preprocessed_data_npy)[indices]
output = np.load(preprossessor.path_preprocessed_data_npy_)[indices]
response = orjson.dumps(output.tolist())
print("Window(s) done: " + str(time() - t0))
......@@ -333,7 +329,7 @@ def table_info():
t0 = time()
raw_data = orjson.loads(request.data)
table = raw_data["table"]
data = np.load(path_preprocessed_data_npy)
data = np.load(preprossessor.path_preprocessed_data_npy_)
response = pseudo.table_info(data, table)
......
backend/src/preprocessing.py
View file @ 33ced1b8
This diff is collapsed.
config.yml
View file @ 33ced1b8
......@@ -3,4 +3,5 @@
reload: false
# Specific configuration file
path_config_yml: D:\Projects\pseudo\experiments\configs\config_egr.yml
\ No newline at end of file
path_config_yml: D:\Projects\pseudo\experiments\configs\config_egr.yml
#path_config_yml: D:\Projects\pseudo\experiments\configs\config_eeg_eye_state.yml
\ No newline at end of file
data/eeg_eye_state/labels.json 0 → 100644
View file @ 33ced1b8
[
{
"time_start": 1.4684558381734427,
"time_end": 6.803324654516323,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 10.435409573402763,
"time_end": 12.794312036851592,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 16.99659523332666,
"time_end": 20.56619266973763,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 22.6517123973563,
"time_end": 22.862607650710995,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 26.104145804125775,
"time_end": 33.99319046665332,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 40.96054476266773,
"time_end": 46.30322451432005,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 51.96615261365912,
"time_end": 70.72020829160824,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 86.74043661125576,
"time_end": 94.3248547967154,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 99.41758461846585,
"time_end": 99.75345483677148,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 101.35469657520528,
"time_end": 101.76086521129581,
"file": "eeg_eye_state.hdf"
},
{
"time_start": 111.04806729421189,
"time_end": 111.61045463649108,
"file": "eeg_eye_state.hdf"
}
]
\ No newline at end of file
data/eeg_eye_state/metadata.json 0 → 100644
View file @ 33ced1b8
{
"short_names": [
"time",
"AF3",
"F7",
"F3",
"FC5",
"T7",
"P7",
"O1",
"O2",
"P8",
"T8",
"FC6",
"F4",
"F8",
"AF4"
],
"long_names": [
"time",
"AF3",
"F7",
"F3",
"FC5",
"T7",
"P7",
"O1",
"O2",
"P8",
"T8",
"FC6",
"F4",
"F8",
"AF4"
],
"units": [
"s",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-",
"-"
]
}
\ No newline at end of file
experiments/configs/config_eeg_eye_state.yml 0 → 100644
View file @ 33ced1b8
---
dataset: eeg_eye_state
path_data_hdf: D:\Projects\pseudo\data\eeg_eye_state\eeg_eye_state.hdf
path_meta_json: D:\Projects\pseudo\data\eeg_eye_state\metadata.json
channels:
- time
- AF3
- F7
- F3
- FC5
- T7
- P7
- O1
- O2
- P8
- T8
- FC6
- F4
- F8
- AF4
dir_in_hdf: resampled
dir_cache: cache\
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment