Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
vig
Sublinear Algorithms for VA
pseudo
Commits
33ced1b8
Commit
33ced1b8
authored
Jul 28, 2021
by
Yuncong Yu
Browse files
Unify input APIs; add EEG eye state dataset.
parent
be2e7008
Changes
7
Expand all
Hide whitespace changes
Inline
Side-by-side
backend/src/main.py
View file @
33ced1b8
import
logging
import
yaml
from
pathlib
import
Path
from
time
import
time
,
perf_counter
from
typing
import
Any
,
Dict
,
List
,
Union
import
numpy
as
np
import
orjson
import
yaml
from
flask
import
Flask
,
request
from
flask_cors
import
CORS
from
src
import
preprocessing
from
src
import
pseudo
from
src.preprocessing
import
Preprocessor
logging
.
basicConfig
(
level
=
logging
.
INFO
)
# Read config
...
...
@@ -27,13 +29,14 @@ def read_config(path_global_config_yml: Union[Path, str] = Path("../config.yml")
config
=
read_config
()
# path_preprocessed_data_npy = 'cache/preprocessed-data.npy'
path_data_hdf
=
Path
(
config
[
"path_data_hdf"
])
path_meta_json
=
Path
(
config
[
"path_meta_json"
])
path_preprocessed_data_npy
=
Path
(
config
[
"dir_cache"
])
/
"preprocessed_data.npy"
channel_names
=
config
[
"channels"
]
dir_in_hdf
=
config
[
"dir_in_hdf"
]
logging
.
basicConfig
(
level
=
logging
.
INFO
)
preprossessor
=
Preprocessor
(
dataset
=
config
[
"dataset"
],
path_data_hdf
=
Path
(
config
[
"path_data_hdf"
]),
path_meta_json
=
Path
(
config
[
"path_meta_json"
]),
channel_names
=
config
[
"channels"
],
dir_in_hdf
=
config
[
"dir_in_hdf"
],
dir_cache
=
Path
(
config
[
"dir_cache"
]),
)
app
=
Flask
(
__name__
)
CORS
(
app
)
...
...
@@ -64,11 +67,7 @@ def read_data():
logging
.
info
(
"Loading data ..."
)
time_start
=
perf_counter
()
# response = preprocessing.read_weather_data()
# response = preprocessing.read_egr_data()
response
=
preprocessing
.
read_data
(
path_data_hdf
=
path_data_hdf
,
path_meta_json
=
path_meta_json
,
channel_names
=
channel_names
,
dir_in_hdf
=
dir_in_hdf
)
response
=
preprossessor
.
create_response
()
response
=
orjson
.
dumps
(
response
)
logging
.
info
(
f
"Completed loading data with
{
perf_counter
()
-
time_start
:
.
2
f
}
second(s)."
)
...
...
@@ -97,11 +96,10 @@ def create_windows():
logging
.
info
(
"Creating window ..."
)
time_start
=
perf_counter
()
if
not
Path
(
path_preprocessed_data_npy
)
.
is_file
():
if
not
preprossessor
.
path_preprocessed_data_npy
_
.
is_file
():
raw_data
=
request
.
json
window_size
=
int
(
raw_data
[
"parameters"
][
"windowsize"
])
# preprocessing.create_eeg_windows(window_size, 5)
preprocessing
.
create_egr_windows
(
window_size
)
preprossessor
.
create_windows
(
window_size
)
logging
.
info
(
f
"Completed windows with
{
perf_counter
()
-
time_start
:
.
2
f
}
second(s)."
)
...
...
@@ -145,7 +143,7 @@ def initialize():
# Read windows
raw_data
=
orjson
.
loads
(
request
.
data
)
data_windowized
=
np
.
load
(
path_preprocessed_data_npy
)
data_windowized
=
np
.
load
(
preprossessor
.
path_preprocessed_data_npy
_
)
data_windowized
=
np
.
swapaxes
(
data_windowized
,
1
,
2
)
# Use a column vector for each channel
# Read the query
...
...
@@ -179,7 +177,7 @@ def get_lsh_parameters():
t0
=
time
()
raw_data
=
orjson
.
loads
(
request
.
data
)
window_size
=
raw_data
[
"windowsize"
]
data
=
np
.
load
(
path_preprocessed_data_npy
)
data
=
np
.
load
(
preprossessor
.
path_preprocessed_data_npy
_
)
data
=
np
.
swapaxes
(
data
,
1
,
2
)
parameters
=
pseudo
.
get_lsh_parameters
(
data
,
window_size
)
...
...
@@ -214,7 +212,7 @@ def update():
"""
t0
=
time
()
raw_data
=
orjson
.
loads
(
request
.
data
)
data
=
np
.
load
(
path_preprocessed_data_npy
)
data
=
np
.
load
(
preprossessor
.
path_preprocessed_data_npy
_
)
data
=
np
.
swapaxes
(
data
,
1
,
2
)
query
=
raw_data
[
"query"
]
query
=
np
.
swapaxes
(
query
,
0
,
1
)
...
...
@@ -247,7 +245,7 @@ def weights():
hash_functions
=
raw_data
[
"hash_functions"
]
query
=
raw_data
[
"query"
]
old_weights
=
raw_data
[
"weights"
]
data
=
np
.
load
(
path_preprocessed_data_npy
)
data
=
np
.
load
(
preprossessor
.
path_preprocessed_data_npy
_
)
new_weights
=
pseudo
.
weights
(
data
,
query
,
old_weights
,
labels
,
hash_functions
)
...
...
@@ -271,17 +269,15 @@ def query():
time_start
=
perf_counter
()
raw_data
=
orjson
.
loads
(
request
.
data
)
# print(raw_data)
start_index
=
raw_data
[
"start_index"
]
query_size
=
raw_data
[
"query_size"
]
window_indices
=
raw_data
[
"indices"
]
if
start_index
is
not
None
:
# preprocessing.create_weather_windows(query_size)
preprocessing
.
create_egr_windows
(
query_size
)
preprossessor
.
create_windows
(
query_size
)
window_indices
=
int
(
start_index
)
data_windowized
=
np
.
load
(
path_preprocessed_data_npy
)
data_windowized
=
np
.
load
(
preprossessor
.
path_preprocessed_data_npy
_
)
response
=
pseudo
.
query
(
data_windowized
,
window_indices
)
response
=
orjson
.
dumps
(
response
)
...
...
@@ -305,7 +301,7 @@ def window():
raw_data
=
orjson
.
loads
(
request
.
data
)
indices
=
raw_data
[
"indices"
]
output
=
np
.
load
(
path_preprocessed_data_npy
)[
indices
]
output
=
np
.
load
(
preprossessor
.
path_preprocessed_data_npy
_
)[
indices
]
response
=
orjson
.
dumps
(
output
.
tolist
())
print
(
"Window(s) done: "
+
str
(
time
()
-
t0
))
...
...
@@ -333,7 +329,7 @@ def table_info():
t0
=
time
()
raw_data
=
orjson
.
loads
(
request
.
data
)
table
=
raw_data
[
"table"
]
data
=
np
.
load
(
path_preprocessed_data_npy
)
data
=
np
.
load
(
preprossessor
.
path_preprocessed_data_npy
_
)
response
=
pseudo
.
table_info
(
data
,
table
)
...
...
backend/src/preprocessing.py
View file @
33ced1b8
This diff is collapsed.
Click to expand it.
config.yml
View file @
33ced1b8
...
...
@@ -3,4 +3,5 @@
reload
:
false
# Specific configuration file
path_config_yml
:
D:\Projects\pseudo\experiments\configs\config_egr.yml
\ No newline at end of file
path_config_yml
:
D:\Projects\pseudo\experiments\configs\config_egr.yml
#path_config_yml: D:\Projects\pseudo\experiments\configs\config_eeg_eye_state.yml
\ No newline at end of file
data/eeg_eye_state/eeg_eye_state.hdf
0 → 100644
View file @
33ced1b8
File added
data/eeg_eye_state/labels.json
0 → 100644
View file @
33ced1b8
[
{
"time_start"
:
1.4684558381734427
,
"time_end"
:
6.803324654516323
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
10.435409573402763
,
"time_end"
:
12.794312036851592
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
16.99659523332666
,
"time_end"
:
20.56619266973763
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
22.6517123973563
,
"time_end"
:
22.862607650710995
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
26.104145804125775
,
"time_end"
:
33.99319046665332
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
40.96054476266773
,
"time_end"
:
46.30322451432005
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
51.96615261365912
,
"time_end"
:
70.72020829160824
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
86.74043661125576
,
"time_end"
:
94.3248547967154
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
99.41758461846585
,
"time_end"
:
99.75345483677148
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
101.35469657520528
,
"time_end"
:
101.76086521129581
,
"file"
:
"eeg_eye_state.hdf"
},
{
"time_start"
:
111.04806729421189
,
"time_end"
:
111.61045463649108
,
"file"
:
"eeg_eye_state.hdf"
}
]
\ No newline at end of file
data/eeg_eye_state/metadata.json
0 → 100644
View file @
33ced1b8
{
"short_names"
:
[
"time"
,
"AF3"
,
"F7"
,
"F3"
,
"FC5"
,
"T7"
,
"P7"
,
"O1"
,
"O2"
,
"P8"
,
"T8"
,
"FC6"
,
"F4"
,
"F8"
,
"AF4"
],
"long_names"
:
[
"time"
,
"AF3"
,
"F7"
,
"F3"
,
"FC5"
,
"T7"
,
"P7"
,
"O1"
,
"O2"
,
"P8"
,
"T8"
,
"FC6"
,
"F4"
,
"F8"
,
"AF4"
],
"units"
:
[
"s"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
,
"-"
]
}
\ No newline at end of file
experiments/configs/config_eeg_eye_state.yml
0 → 100644
View file @
33ced1b8
---
dataset
:
eeg_eye_state
path_data_hdf
:
D:\Projects\pseudo\data\eeg_eye_state\eeg_eye_state.hdf
path_meta_json
:
D:\Projects\pseudo\data\eeg_eye_state\metadata.json
channels
:
-
time
-
AF3
-
F7
-
F3
-
FC5
-
T7
-
P7
-
O1
-
O2
-
P8
-
T8
-
FC6
-
F4
-
F8
-
AF4
dir_in_hdf
:
resampled
dir_cache
:
cache\
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment