Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
vig
Sublinear Algorithms for VA
pseudo
Commits
6ef8f879
Commit
6ef8f879
authored
Jul 27, 2021
by
Yuncong Yu
Browse files
Sort data directory.
parent
bfd51f4c
Changes
4
Hide whitespace changes
Inline
Side-by-side
.gitignore
View file @
6ef8f879
...
...
@@ -10,6 +10,7 @@ __pycache__
201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01.h5
201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5
Angu:arApp/prototype/node_modules
backend/cache/
clion-log.txt
Documents/
Flaskserver/venv/*
...
...
backend/src/main.py
View file @
6ef8f879
...
...
@@ -13,7 +13,7 @@ from src import pseudo
# Config
path_preprocessed_data_npy
=
'
data
/processed-data.npy'
path_preprocessed_data_npy
=
'
cache
/processed-data.npy'
reload
=
False
logging
.
basicConfig
(
level
=
logging
.
INFO
)
...
...
backend/src/preprocessing.py
View file @
6ef8f879
...
...
@@ -12,7 +12,7 @@ import tables
from
sklearn.preprocessing
import
minmax_scale
logging
.
basicConfig
(
level
=
logging
.
INFO
)
data_path
=
"
data
/processed-data.npy"
data_path
=
"
cache
/processed-data.npy"
# def read_data():
...
...
@@ -51,7 +51,7 @@ def create_peax_windows_12kb_mts(window_size):
def
read_eeg_data
(
nr_of_channels
):
response
=
[]
datafile
=
"data/21.csv"
datafile
=
"
../
data/21.csv"
data
=
pd
.
read_csv
(
datafile
,
header
=
None
)
npdata
=
np
.
array
(
data
,
dtype
=
"float32"
)
del
data
...
...
@@ -63,8 +63,8 @@ def read_eeg_data(nr_of_channels):
def
create_eeg_windows
(
window_size
,
nr_of_channels
):
data_path
=
"data/processed-data_"
+
str
(
window_size
)
+
".npy"
datafile
=
"data/21.csv"
data_path
=
"
../
data/processed-data_"
+
str
(
window_size
)
+
".npy"
datafile
=
"
../
data/21.csv"
if
not
os
.
path
.
isfile
(
data_path
):
data
=
pd
.
read_csv
(
datafile
,
header
=
None
)
...
...
@@ -79,13 +79,13 @@ def create_eeg_windows(window_size, nr_of_channels):
del
npdata
data
=
np
.
reshape
(
np_window_data
,
(
len
(
np_window_data
),
nr_of_channels
,
len
(
np_window_data
[
0
])))
np
.
save
(
data_path
,
data
)
np
.
save
(
"data/processed-data"
,
np
.
load
(
data_path
))
np
.
save
(
"
../
data/processed-data"
,
np
.
load
(
data_path
))
return
"1"
def
read_eeg_data
(
nr_of_channels
):
response
=
[]
datafile
=
"data/21.csv"
datafile
=
"
../
data/21.csv"
data
=
pd
.
read_csv
(
datafile
,
header
=
None
)
npdata
=
np
.
array
(
data
,
dtype
=
"float32"
)
del
data
...
...
@@ -100,7 +100,7 @@ def read_weather_data():
filename
=
"../data/weather.pkl"
if
not
os
.
path
.
isfile
(
filename
):
print
(
"start"
)
df
=
dd
.
read_csv
(
"data/NW_Ground_Stations_2016.csv"
,
usecols
=
[
"number_sta"
,
"date"
,
"t"
,
"hu"
,
"td"
,
"dd"
,
"ff"
,
"psl"
,
"precip"
])
df
=
dd
.
read_csv
(
"
../
data/NW_Ground_Stations_2016.csv"
,
usecols
=
[
"number_sta"
,
"date"
,
"t"
,
"hu"
,
"td"
,
"dd"
,
"ff"
,
"psl"
,
"precip"
])
print
(
"read file"
)
df
=
df
.
loc
[
df
[
"number_sta"
].
isin
([
14066001
,
14137001
,
14216001
,
14372001
,
22092001
,
22113006
,
22135001
])].
fillna
(
0
)
df
[
"date"
]
=
dd
.
to_datetime
(
df
[
"date"
],
format
=
"%Y%m%d %H:%M"
)
...
...
@@ -151,7 +151,7 @@ def read_weather_data():
def
create_weather_windows
(
window_size
):
if
not
os
.
path
.
isfile
(
"
data
/weather-"
+
str
(
window_size
)
+
".npy"
):
if
not
os
.
path
.
isfile
(
"
cache
/weather-"
+
str
(
window_size
)
+
".npy"
):
filename
=
"../data/weather.pkl"
df
=
pd
.
read_pickle
(
filename
)
channels
=
list
()
...
...
@@ -170,9 +170,9 @@ def create_weather_windows(window_size):
windows
.
append
(
minmax_scale
(
data
[
i
],
(
-
1
,
1
),
axis
=
1
))
print
(
"dims:"
)
print
(
windows
[
0
].
size
)
np
.
save
(
"data/weather-"
+
str
(
window_size
),
windows
)
data
=
np
.
load
(
"data/weather-"
+
str
(
window_size
)
+
".npy"
)
np
.
save
(
"data/processed-data"
,
data
)
np
.
save
(
"
../
data/weather-"
+
str
(
window_size
),
windows
)
data
=
np
.
load
(
"
../
data/weather-"
+
str
(
window_size
)
+
".npy"
)
np
.
save
(
"
../
data/processed-data"
,
data
)
return
"1"
...
...
@@ -181,7 +181,7 @@ def read_egr_data():
# Config
path_data_original
:
Union
[
Path
,
str
]
=
"data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
]
=
"
../
data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
channel_names
=
[
"time"
,
"ACM_Egrrate_demand_managed"
,
"ACM_Egrrate_feedback_filt"
,
"ACM_Egr_enable"
]
# Load data
...
...
@@ -203,9 +203,9 @@ def read_egr_data():
def
create_egr_windows
(
window_size
):
"""Create windows for EGR dataset."""
# Config
path_data_original_hdf
=
"data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
path_data_cached_npy
=
f
"
data
/egr_cached_
{
window_size
}
.npy"
path_data_preprocessed_npy
=
f
"
data
/processed-data.npy"
path_data_original_hdf
=
"
../
data/201207_IAVHeKu_212-SM-9221_WMA4ID41_DS18_TestV_10_EU5FM_800m_0C_freie_Fahrt_nrm_01_compressed.h5"
path_data_cached_npy
=
f
"
cache
/egr_cached_
{
window_size
}
.npy"
path_data_preprocessed_npy
=
f
"
cache
/processed-data.npy"
# Created cached data
if
not
Path
(
path_data_cached_npy
).
is_file
():
...
...
backend/src/pseudo.py
View file @
6ef8f879
...
...
@@ -16,10 +16,10 @@ def get_lsh_parameters(data, window_size):
"""
data: 3d array [m][t][d]
"""
if
(
not
os
.
path
.
isfile
(
'
data
/parameters-'
+
str
(
window_size
)
+
'.npy'
)):
if
(
not
os
.
path
.
isfile
(
'
cache
/parameters-'
+
str
(
window_size
)
+
'.npy'
)):
parameters
=
preprocess
(
data
)
np
.
save
(
'
data
/parameters-'
+
str
(
window_size
),
[
float
(
parameters
[
0
]),
float
(
parameters
[
1
]),
float
(
parameters
[
2
])])
return
np
.
load
(
'
data
/parameters-'
+
str
(
window_size
)
+
'.npy'
).
tolist
()
np
.
save
(
'
cache
/parameters-'
+
str
(
window_size
),
[
float
(
parameters
[
0
]),
float
(
parameters
[
1
]),
float
(
parameters
[
2
])])
return
np
.
load
(
'
cache
/parameters-'
+
str
(
window_size
)
+
'.npy'
).
tolist
()
def
lsh
(
data
,
query
,
parameters
=
None
,
weights
=
None
):
...
...
@@ -269,7 +269,7 @@ def query(data, window_indices):
def
debug_test_lsh
():
data
=
np
.
load
(
'
../data
/processed-data.npy'
)
data
=
np
.
load
(
'
cache
/processed-data.npy'
)
# data = np.repeat(data, repeats=7, axis=1)
print
(
data
.
shape
)
data
=
np
.
reshape
(
data
,
(
len
(
data
),
len
(
data
[
0
][
0
]),
len
(
data
[
0
])))
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment