Commit aea9dde7 authored by Yuncong Yu's avatar Yuncong Yu
Browse files

Update backend for TP's data in IAV

- Update backend for TP's data;
- Time format from datetime to second;
- Unittest for some backend functions;
- IAV color;
parent 291d8bec
...@@ -41,8 +41,8 @@ ...@@ -41,8 +41,8 @@
} }
.correct-selected { .correct-selected {
background-color: #4caf50 background-color: #91c60e /*#4caf50*/
} }
.neutral-selected { background-color: #ffa300 .neutral-selected { background-color: #ffc600 /*#ffa300*/
} }
.incorrect-selected { background-color: #f44336 } .incorrect-selected { background-color: #ce0037 /*#f44336*/ }
...@@ -84,7 +84,7 @@ mat-tab-group { ...@@ -84,7 +84,7 @@ mat-tab-group {
.channel_header { .channel_header {
height: 18px; height: 18px;
background-color: lightblue; background-color: #14639e /*lightblue*/;
border-bottom: 1px solid lightgray; border-bottom: 1px solid lightgray;
color: white; color: white;
text-align: center; text-align: center;
......
...@@ -108,7 +108,7 @@ path { ...@@ -108,7 +108,7 @@ path {
} }
.channel_header { .channel_header {
background-color: lightblue; background-color: #14639e /*lightblue*/;
border-bottom: 1px solid lightgray; border-bottom: 1px solid lightgray;
color: white; color: white;
text-align: center; text-align: center;
......
...@@ -147,11 +147,11 @@ export class OverviewWindowComponent implements OnInit { ...@@ -147,11 +147,11 @@ export class OverviewWindowComponent implements OnInit {
scaleX: { scaleX: {
zooming: true, zooming: true,
'min-value': channel.index[0], 'min-value': channel.index[0],
step: "6minute", // step: "6minute",
transform: { // transform: {
type: "date", // type: "date",
all: "%m/%d/%Y<br>%h:%i" // all: "%m/%d/%Y<br>%h:%i"
}, // },
tick: { tick: {
visible: index === 0, visible: index === 0,
}, },
...@@ -249,11 +249,11 @@ export class OverviewWindowComponent implements OnInit { ...@@ -249,11 +249,11 @@ export class OverviewWindowComponent implements OnInit {
scaleX: { scaleX: {
zooming: true, zooming: true,
'min-value': channel.index[0], 'min-value': channel.index[0],
step: "6minute", // step: "6minute",
transform: { // transform: {
type: "date", // type: "date",
all: "%m/%d/%Y<br>%h:%i" // all: "%m/%d/%Y<br>%h:%i"
}, // },
tick: { tick: {
visible: index === 0, visible: index === 0,
}, },
......
...@@ -5,11 +5,11 @@ ...@@ -5,11 +5,11 @@
.train-button:hover { .train-button:hover {
background-color: white; background-color: white;
color: black; color: black;
border: 1px solid #4CAF50; border: 3px solid #14639e /*#4CAF50*/;
} }
.train-button { .train-button {
background-color: #4CAF50; background-color: #14639e /*#4CAF50*/;
color: white; color: white;
} }
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
} }
button { button {
background-color: #4CAF50; background-color: #14639e /*#4CAF50*/;
border: none; border: none;
color: white; color: white;
padding: 10px 30px; padding: 10px 30px;
...@@ -29,6 +29,7 @@ button { ...@@ -29,6 +29,7 @@ button {
text-decoration: none; text-decoration: none;
display: inline-block; display: inline-block;
font-size: 16px; font-size: 16px;
font-weight: bold;
margin: 4px 2px; margin: 4px 2px;
transition-duration: 0.4s; transition-duration: 0.4s;
cursor: pointer; cursor: pointer;
......
from flask import Flask, request import logging
import os
from pathlib import Path
from time import time, perf_counter
from typing import Any, Dict, List
import numpy as np import numpy as np
from flask_cors import CORS
from time import time
import orjson import orjson
import os.path from flask import Flask, request
import pseudo from flask_cors import CORS
import preprocessing import preprocessing
import pseudo
data_path = 'data/processed-data.npy'
# Config
path_preprocessed_data_npy = 'data/processed-data.npy'
reload = False reload = False
logging.basicConfig(level=logging.INFO)
app = Flask(__name__) app = Flask(__name__)
CORS(app) CORS(app)
@app.route('/', methods=['GET']) @app.route('/', methods=['GET'])
def index(): def index():
return "hi" return "hi"
"""
Returns raw data
Output: [{
index: 1d array [x]
values: 1d array [x]
}]
"""
@app.route('/read-data', methods=['GET']) @app.route('/read-data', methods=['GET'])
def read_data(): def read_data():
t0 = time() """
response = preprocessing.read_weather_data() Load raw data.
Returns
-------
response : List[Dict[str, np.ndarray]]
Loaded data with the following interface
{
index: 1d array [x]
values: 1d array [x]
name: str
}[]
"""
logging.info('Loading data ...')
time_start = perf_counter()
# response = preprocessing.read_weather_data()
response = preprocessing.read_egr_data()
response = orjson.dumps(response) response = orjson.dumps(response)
print('Data read: ' + str(time()-t0))
return response
logging.info(f'Completed loading data with {perf_counter() - time_start:.2f} second(s).')
""" return response
Creates windows
Input: {
parameters: {
windowssize: int
}
}
Output: '1'
"""
@app.route('/create-windows', methods=['POST']) @app.route('/create-windows', methods=['POST'])
def create_windows(): def create_windows():
t0 = time() """
if (not os.path.isfile(data_path)): Creates windows to transform the local pattern search problem to time series indexing.
Returns
-------
'1'
Use dynamic input from request with the interface
{
parameters: {
windowssize: int
}
}
"""
logging.info('Creating window ...')
time_start = perf_counter()
if not Path(path_preprocessed_data_npy).is_file():
raw_data = request.json raw_data = request.json
window_size = int(raw_data['parameters']["windowsize"]) window_size = int(raw_data['parameters']["windowsize"])
preprocessing.create_eeg_windows(window_size, 5) # preprocessing.create_eeg_windows(window_size, 5)
print('Windows created: ' + str(time()-t0)) preprocessing.create_egr_windows(window_size)
logging.info(f'Completed windows with {perf_counter() - time_start:.2f} second(s).')
return '1' return '1'
"""
Does first iteration of LSH and returns a bunch of useful information
Input: {
query: 2d array [d][t]
}
Output: {
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
parameters: 1d array
}
"""
@app.route('/initialize', methods=['POST']) @app.route('/initialize', methods=['POST'])
def initialize(): def initialize():
t0 = time() """
Conduct the initial LSH.
Returns
-------
response : Dict[str, Any]
Response with the interface
{
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
parameters: 1d array
}
Use dynamic input from request have the interface
{
query: 2d array [d][t]
}
"""
logging.info("Starting the initial LSH ...")
time_start = perf_counter()
# Read windows
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
data = np.load(data_path) data_windowized = np.load(path_preprocessed_data_npy)
data = np.swapaxes(data, 1, 2) data_windowized = np.swapaxes(data_windowized, 1, 2) # Use a column vector for each channel
# Read the query
query = raw_data["query"] query = raw_data["query"]
query = np.swapaxes(query, 0, 1) query = np.swapaxes(query, 0, 1)
# parameters = np.load('parameters.npy') # parameters = np.load('parameters.npy')
lsh_data = pseudo.lsh(data, query) # Run the initial LSH
logging.info("Executing the initial LSH ...")
lsh_data = pseudo.lsh(data_windowized, query)
response = orjson.dumps(lsh_data) response = orjson.dumps(lsh_data)
print('LSH done: ' + str(time()-t0)) logging.info(f'Completed the initial LSH with {perf_counter() - time_start:2f} second(s)')
return response
""" return response
Calculates LSH parameters based on the dataset
Input: {
windowsize: int
}
Output: {
parameters: 1d array [3]
}
"""
@app.route('/get-lsh-parameters', methods=['POST']) @app.route('/get-lsh-parameters', methods=['POST'])
def get_lsh_parameters(): def get_lsh_parameters():
"""
Calculates LSH parameters based on the dataset
Input: {
windowsize: int
}
Output: {
parameters: 1d array [3]
}
"""
t0 = time() t0 = time()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
window_size = raw_data["windowsize"] window_size = raw_data["windowsize"]
data = np.load(data_path) data = np.load(path_preprocessed_data_npy)
data = np.swapaxes(data, 1, 2) data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data, window_size) parameters = pseudo.get_lsh_parameters(data, window_size)
response = orjson.dumps(parameters) response = orjson.dumps(parameters)
print('Parameter calculation done: ' + str(time()-t0)) print('Parameter calculation done: ' + str(time() - t0))
return response return response
"""
Does LSH and returns a bunch of useful information
Input: {
query: 2d array [d][t]
}
Output: {
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
}
"""
@app.route('/update', methods=['POST']) @app.route('/update', methods=['POST'])
def update(): def update():
"""
Does LSH and returns a bunch of useful information
Input: {
query: 2d array [d][t]
}
Output: {
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
}
"""
t0 = time() t0 = time()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
data = np.load(data_path) data = np.load(path_preprocessed_data_npy)
data = np.swapaxes(data, 1, 2) data = np.swapaxes(data, 1, 2)
query = raw_data["query"] query = raw_data["query"]
query = np.swapaxes(query, 0, 1) query = np.swapaxes(query, 0, 1)
...@@ -158,30 +204,30 @@ def update(): ...@@ -158,30 +204,30 @@ def update():
lsh_data = pseudo.lsh(data, query, parameters=parameters, weights=weights) lsh_data = pseudo.lsh(data, query, parameters=parameters, weights=weights)
response = orjson.dumps(lsh_data) response = orjson.dumps(lsh_data)
print('LSH done: ' + str(time()-t0)) print('LSH done: ' + str(time() - t0))
return response return response
"""
Calculates new weights for LSH algorithm
Input: {
labels: 1d array [?]
hash_functions: 2d array [?][d]
query: 2d array [d][t]
weights: 1d array [d]
}
Output: 1d array [d]
"""
@app.route('/weights', methods=['POST']) @app.route('/weights', methods=['POST'])
def weights(): def weights():
"""
Calculates new weights for LSH algorithm
Input: {
labels: 1d array [?]
hash_functions: 2d array [?][d]
query: 2d array [d][t]
weights: 1d array [d]
}
Output: 1d array [d]
"""
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
labels = raw_data["labels"] labels = raw_data["labels"]
hash_functions = raw_data["hash_functions"] hash_functions = raw_data["hash_functions"]
query = raw_data["query"] query = raw_data["query"]
old_weights = raw_data["weights"] old_weights = raw_data["weights"]
data = np.load(data_path) data = np.load(path_preprocessed_data_npy)
new_weights = pseudo.weights(data, query, old_weights, labels, hash_functions) new_weights = pseudo.weights(data, query, old_weights, labels, hash_functions)
...@@ -189,81 +235,87 @@ def weights(): ...@@ -189,81 +235,87 @@ def weights():
return response return response
"""
Calculates query based on given indices
Input: {
indices: 1d array [?]
}
Output: 2d array [d][t]
"""
@app.route('/query', methods=['POST']) @app.route('/query', methods=['POST'])
def query(): def query():
t0 = time() """
Calculate the query based on the given indices.
Input: {
indices: 1d array [?]
}
Output: 2d array [d][t]
"""
logging.info("Preparing the query ...")
time_start = perf_counter()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
print(raw_data) # print(raw_data)
start_index = raw_data['start_index'] start_index = raw_data['start_index']
query_size = raw_data['query_size'] query_size = raw_data['query_size']
window_indices = raw_data['indices'] window_indices = raw_data['indices']
if start_index is not None: if start_index is not None:
preprocessing.create_weather_windows(query_size) # preprocessing.create_weather_windows(query_size)
preprocessing.create_egr_windows(query_size)
window_indices = int(start_index) window_indices = int(start_index)
data = np.load(data_path)
response = pseudo.query(data, window_indices)
response = orjson.dumps(response) data_windowized = np.load(path_preprocessed_data_npy)
print("Query done: " + str(time() - t0)) response = pseudo.query(data_windowized, window_indices)
return response
response = orjson.dumps(response)
logging.info(f"Completed preparing the query with {time() - time_start}.")
""" return response
Returns values of windows on given indices
Input: {
indices: 1d array [x]
}
Output: 3d array [x][d][t]
"""
@app.route('/window', methods=['POST']) @app.route('/window', methods=['POST'])
def window(): def window():
"""
Returns values of windows on given indices
Input: {
indices: 1d array [x]
}
Output: 3d array [x][d][t]
"""
t0 = time() t0 = time()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)