Skip to content

GitLab

Next tuesday 25th january around 21.30 we'll be upgrading to GitLab version 14.7

Commit aea9dde7 authored by Yuncong Yu's avatar Yuncong Yu
Browse files

Update backend for TP's data in IAV 

- Update backend for TP's data;
- Time format from datetime to second;
- Unittest for some backend functions;
- IAV color;
parent 291d8bec
Hide whitespace changes
Inline Side-by-side
Showing with 425 additions and 298 deletions
AngularApp/prototype/src/app/labeling-window/labeling-window.component.css
View file @ aea9dde7
......@@ -41,8 +41,8 @@
}
.correct-selected {
background-color: #4caf50
background-color: #91c60e /*#4caf50*/
}
.neutral-selected { background-color: #ffa300
.neutral-selected { background-color: #ffc600 /*#ffa300*/
}
.incorrect-selected { background-color: #f44336 }
.incorrect-selected { background-color: #ce0037 /*#f44336*/ }
AngularApp/prototype/src/app/main/main.component.css
View file @ aea9dde7
......@@ -84,7 +84,7 @@ mat-tab-group {
.channel_header {
height: 18px;
background-color: lightblue;
background-color: #14639e /*lightblue*/;
border-bottom: 1px solid lightgray;
color: white;
text-align: center;
......
AngularApp/prototype/src/app/overview-window/overview-window.component.css
View file @ aea9dde7
......@@ -108,7 +108,7 @@ path {
}
.channel_header {
background-color: lightblue;
background-color: #14639e /*lightblue*/;
border-bottom: 1px solid lightgray;
color: white;
text-align: center;
......
AngularApp/prototype/src/app/overview-window/overview-window.component.ts
View file @ aea9dde7
......@@ -147,11 +147,11 @@ export class OverviewWindowComponent implements OnInit {
scaleX: {
zooming: true,
'min-value': channel.index[0],
step: "6minute",
transform: {
type: "date",
all: "%m/%d/%Y<br>%h:%i"
},
// step: "6minute",
// transform: {
// type: "date",
// all: "%m/%d/%Y<br>%h:%i"
// },
tick: {
visible: index === 0,
},
......@@ -249,11 +249,11 @@ export class OverviewWindowComponent implements OnInit {
scaleX: {
zooming: true,
'min-value': channel.index[0],
step: "6minute",
transform: {
type: "date",
all: "%m/%d/%Y<br>%h:%i"
},
// step: "6minute",
// transform: {
// type: "date",
// all: "%m/%d/%Y<br>%h:%i"
// },
tick: {
visible: index === 0,
},
......
AngularApp/prototype/src/app/training-window/training-window.component.css
View file @ aea9dde7
......@@ -5,11 +5,11 @@
.train-button:hover {
background-color: white;
color: black;
border: 1px solid #4CAF50;
border: 3px solid #14639e /*#4CAF50*/;
}
.train-button {
background-color: #4CAF50;
background-color: #14639e /*#4CAF50*/;
color: white;
}
......@@ -21,7 +21,7 @@
}
button {
background-color: #4CAF50;
background-color: #14639e /*#4CAF50*/;
border: none;
color: white;
padding: 10px 30px;
......@@ -29,6 +29,7 @@ button {
text-decoration: none;
display: inline-block;
font-size: 16px;
font-weight: bold;
margin: 4px 2px;
transition-duration: 0.4s;
cursor: pointer;
......
Flaskserver/main.py
View file @ aea9dde7
from flask import Flask, request
import logging
import os
from pathlib import Path
from time import time, perf_counter
from typing import Any, Dict, List
import numpy as np
from flask_cors import CORS
from time import time
import orjson
import os.path
import pseudo
from flask import Flask, request
from flask_cors import CORS
import preprocessing
import pseudo
data_path = 'data/processed-data.npy'
# Config
path_preprocessed_data_npy = 'data/processed-data.npy'
reload = False
logging.basicConfig(level=logging.INFO)
app = Flask(__name__)
CORS(app)
@app.route('/', methods=['GET'])
def index():
return "hi"
"""
Returns raw data
Output: [{
index: 1d array [x]
values: 1d array [x]
}]
"""
@app.route('/read-data', methods=['GET'])
def read_data():
t0 = time()
response = preprocessing.read_weather_data()
"""
Load raw data.
Returns
-------
response : List[Dict[str, np.ndarray]]
Loaded data with the following interface
{
index: 1d array [x]
values: 1d array [x]
name: str
}[]
"""
logging.info('Loading data ...')
time_start = perf_counter()
# response = preprocessing.read_weather_data()
response = preprocessing.read_egr_data()
response = orjson.dumps(response)
print('Data read: ' + str(time()-t0))
return response
logging.info(f'Completed loading data with {perf_counter() - time_start:.2f} second(s).')
"""
Creates windows
return response
Input: {
parameters: {
windowssize: int
}
}
Output: '1'
"""
@app.route('/create-windows', methods=['POST'])
def create_windows():
t0 = time()
if (not os.path.isfile(data_path)):
"""
Creates windows to transform the local pattern search problem to time series indexing.
Returns
-------
'1'
Use dynamic input from request with the interface
{
parameters: {
windowssize: int
}
}
"""
logging.info('Creating window ...')
time_start = perf_counter()
if not Path(path_preprocessed_data_npy).is_file():
raw_data = request.json
window_size = int(raw_data['parameters']["windowsize"])
preprocessing.create_eeg_windows(window_size, 5)
print('Windows created: ' + str(time()-t0))
# preprocessing.create_eeg_windows(window_size, 5)
preprocessing.create_egr_windows(window_size)
logging.info(f'Completed windows with {perf_counter() - time_start:.2f} second(s).')
return '1'
"""
Does first iteration of LSH and returns a bunch of useful information
Input: {
query: 2d array [d][t]
}
Output: {
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
parameters: 1d array
}
"""
@app.route('/initialize', methods=['POST'])
def initialize():
t0 = time()
"""
Conduct the initial LSH.
Returns
-------
response : Dict[str, Any]
Response with the interface
{
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
parameters: 1d array
}
Use dynamic input from request have the interface
{
query: 2d array [d][t]
}
"""
logging.info("Starting the initial LSH ...")
time_start = perf_counter()
# Read windows
raw_data = orjson.loads(request.data)
data = np.load(data_path)
data = np.swapaxes(data, 1, 2)
data_windowized = np.load(path_preprocessed_data_npy)
data_windowized = np.swapaxes(data_windowized, 1, 2) # Use a column vector for each channel
# Read the query
query = raw_data["query"]
query = np.swapaxes(query, 0, 1)
# parameters = np.load('parameters.npy')
lsh_data = pseudo.lsh(data, query)
# Run the initial LSH
logging.info("Executing the initial LSH ...")
lsh_data = pseudo.lsh(data_windowized, query)
response = orjson.dumps(lsh_data)
print('LSH done: ' + str(time()-t0))
return response
logging.info(f'Completed the initial LSH with {perf_counter() - time_start:2f} second(s)')
"""
Calculates LSH parameters based on the dataset
return response
Input: {
windowsize: int
}
Output: {
parameters: 1d array [3]
}
"""
@app.route('/get-lsh-parameters', methods=['POST'])
def get_lsh_parameters():
"""
Calculates LSH parameters based on the dataset
Input: {
windowsize: int
}
Output: {
parameters: 1d array [3]
}
"""
t0 = time()
raw_data = orjson.loads(request.data)
window_size = raw_data["windowsize"]
data = np.load(data_path)
data = np.load(path_preprocessed_data_npy)
data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data, window_size)
response = orjson.dumps(parameters)
print('Parameter calculation done: ' + str(time()-t0))
print('Parameter calculation done: ' + str(time() - t0))
return response
"""
Does LSH and returns a bunch of useful information
Input: {
query: 2d array [d][t]
}
Output: {
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
}
"""
@app.route('/update', methods=['POST'])
def update():
"""
Does LSH and returns a bunch of useful information
Input: {
query: 2d array [d][t]
}
Output: {
hash_functions: 3d array [k][l][d]
candidates: 3d array [k][l][i]
distances: 3d array [k][l][i]
average_candidates: 1d array [i]
average_distances: 1d array [i]
tables: [{
bucket: 1d array
}]
average_table: {
bucket: 1d array
}
samples: 1d array
}
"""
t0 = time()
raw_data = orjson.loads(request.data)
data = np.load(data_path)
data = np.load(path_preprocessed_data_npy)
data = np.swapaxes(data, 1, 2)
query = raw_data["query"]
query = np.swapaxes(query, 0, 1)
......@@ -158,30 +204,30 @@ def update():
lsh_data = pseudo.lsh(data, query, parameters=parameters, weights=weights)
response = orjson.dumps(lsh_data)
print('LSH done: ' + str(time()-t0))
print('LSH done: ' + str(time() - t0))
return response
"""
Calculates new weights for LSH algorithm
Input: {
labels: 1d array [?]
hash_functions: 2d array [?][d]
query: 2d array [d][t]
weights: 1d array [d]
}
Output: 1d array [d]
"""
@app.route('/weights', methods=['POST'])
def weights():
"""
Calculates new weights for LSH algorithm
Input: {
labels: 1d array [?]
hash_functions: 2d array [?][d]
query: 2d array [d][t]
weights: 1d array [d]
}
Output: 1d array [d]
"""
raw_data = orjson.loads(request.data)
labels = raw_data["labels"]
hash_functions = raw_data["hash_functions"]
query = raw_data["query"]
old_weights = raw_data["weights"]
data = np.load(data_path)
data = np.load(path_preprocessed_data_npy)
new_weights = pseudo.weights(data, query, old_weights, labels, hash_functions)
......@@ -189,81 +235,87 @@ def weights():
return response
"""
Calculates query based on given indices
Input: {
indices: 1d array [?]
}
Output: 2d array [d][t]
"""
@app.route('/query', methods=['POST'])
def query():
t0 = time()
"""
Calculate the query based on the given indices.
Input: {
indices: 1d array [?]
}
Output: 2d array [d][t]
"""
logging.info("Preparing the query ...")
time_start = perf_counter()
raw_data = orjson.loads(request.data)
print(raw_data)
# print(raw_data)
start_index = raw_data['start_index']
query_size = raw_data['query_size']
window_indices = raw_data['indices']
if start_index is not None:
preprocessing.create_weather_windows(query_size)
# preprocessing.create_weather_windows(query_size)
preprocessing.create_egr_windows(query_size)
window_indices = int(start_index)
data = np.load(data_path)
response = pseudo.query(data, window_indices)
response = orjson.dumps(response)
print("Query done: " + str(time() - t0))
return response
data_windowized = np.load(path_preprocessed_data_npy)
response = pseudo.query(data_windowized, window_indices)
response = orjson.dumps(response)
logging.info(f"Completed preparing the query with {time() - time_start}.")
"""
Returns values of windows on given indices
return response
Input: {
indices: 1d array [x]
}
Output: 3d array [x][d][t]
"""
@app.route('/window', methods=['POST'])
def window():
"""
Returns values of windows on given indices
Input: {
indices: 1d array [x]
}
Output: 3d array [x][d][t]
"""
t0 = time()
raw_data = orjson.loads(request.data)
indices = raw_data['indices']
output = np.load(data_path)[indices]
output = np.load(path_preprocessed_data_npy)[indices]
response = orjson.dumps(output.tolist())
print("Window(s) done: " + str(time() - t0))
return response
"""
Returns additional information on given table
@app.route('/table-info', methods=['POST'])
def table_info():
"""
Returns additional information on given table
Input: {
table: 2d array [x][?]
}
Input: {
table: 2d array [x][?]
}
Output: {
prototypes: {
average: 1d array [t]
max: 1d array [t]
min: 1d array [t]
Output: {
prototypes: {
average: 1d array [t]
max: 1d array [t]
min: 1d array [t]
}
distances: 2d array [x][x]
}
distances: 2d array [x][x]
}
"""
@app.route('/table-info', methods=['POST'])
def table_info():
"""
t0 = time()
raw_data = orjson.loads(request.data)
table = raw_data['table']
data = np.load(data_path)
data = np.load(path_preprocessed_data_npy)
response = pseudo.table_info(data, table)
print("Averages calculated: " + str(time() - t0))
return response
\ No newline at end of file
return response
Flaskserver/preprocessing.py
View file @ aea9dde7
import numpy as np
import pandas as pd
import logging
import os.path
from pathlib import Path
from typing import Union
# from libs import bigwig
# import bbi
import dask.dataframe as dd
import os.path
from sklearn import preprocessing
import numpy as np
import pandas as pd
import tables
from sklearn.preprocessing import minmax_scale
data_path = 'data/processed-data.npy'
logging.basicConfig(level=logging.INFO)
data_path = "data/processed-data.npy"
def read_data():
size = bbi.chromsizes('test.bigWig')['chr1']
size = bbi.chromsizes("test.bigWig")["chr1"]
bins = 100000
data = bigwig.get('data/test.bigWig', 'chr1', 0, size, bins)
data = bigwig.get("data/test.bigWig", "chr1", 0, size, bins)
print(data.shape)
response = [
{
"index": list(range(0, size, int(size/(bins)))),
"values": data.tolist()
},
{
"index": list(range(0, size, int(size / (bins)))),
"values": data.tolist()
},
{
"index": list(range(0, size, int(size / (bins)))),
"values": data.tolist()
}
{"index": list(range(0, size, int(size / (bins)))), "values": data.tolist()},
{"index": list(range(0, size, int(size / (bins)))), "values": data.tolist()},
{"index": list(range(0, size, int(size / (bins)))), "values": data.tolist()},
]
return response
def create_peax_windows_12kb(window_size):
data = bigwig.chunk(
'test.bigWig',
12000,
int(12000 / window_size),
int(12000 / 6),