Skip to content

GitLab

Commit 6e893f4b authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Reintroduced MTS data

parent 9353b768
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 1724 additions and 930 deletions
AngularApp/prototype/src/app/api.service.ts
View file @ 6e893f4b
import { Injectable } from '@angular/core';
export interface RawData {
index: string[];
values: number[];
index: string[][];
values: number[][];
}
export interface LshData {
......@@ -29,15 +29,15 @@ export class ApiService {
constructor() { }
// Read input data
async readFile(): Promise<RawData> {
const response = await fetch('http://127.0.0.1:5000/read-data');
async readFile(): Promise<RawData[]> {
const response = await fetch('http://127.0.0.1:5000/read-mts-data');
return await response.json();
}
// Split data into windows and normalize
async createWindows(parameters): Promise<any> {
const postData = {parameters};
const response = await fetch('http://127.0.0.1:5000/create-windows', {
const response = await fetch('http://127.0.0.1:5000/create-mts-windows', {
method: 'POST',
headers: {
'Accept': 'application/json',
......@@ -74,7 +74,7 @@ export class ApiService {
}
// Get query window based on windows labeled correct
async getQueryWindow(window): Promise<number[]> {
async getQueryWindow(window): Promise<number[][]> {
const response = await fetch('http://127.0.0.1:5000/query', {
method: 'POST',
headers: {
......@@ -87,7 +87,7 @@ export class ApiService {
}
// Get data of a window by indices
async getWindowByIndices(indices: number[]): Promise<number[][]> {
async getWindowByIndices(indices: number[]): Promise<number[][][]> {
const response = await fetch('http://127.0.0.1:5000/window', {
method: 'POST',
headers: {
......
AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts
View file @ 6e893f4b
......@@ -48,45 +48,67 @@ export class LabelingWindowComponent implements OnInit {
.filter((candidate) => this.state.labels[candidate] !== true)
.slice(0, this.k);
this.subplots = [];
const values = await this.state.getWindow(this.topk);
this.topk.forEach((index, i) => {
this.subplots.push(
{
index,
data: [{
x: [...Array(values[i].length).keys()],
y: values[i],
type: 'line'
}],
layout: {
title: `Index: ${index.toString()}`,
hovermode: 'closest',
autosize: true,
margin: {
l: 30,
r: 30,
t: 30,
b: 0,
pad: 4
},
height: 150,
width: 300,
titlefont: {
size: 9
},
xaxis: {
showgrid: false,
zeroline: false,
showticklabels: false,
},
yaxis: {
zeroline: false,
showticklabels: false,
}
const values: number[][][] = await this.state.getWindow(this.topk);
for (const idx in this.topk) {
const window = values[idx].slice(0, 1);
const data = [];
window.forEach((channel: number[], index: number) => {
data.push({
x: [...Array(channel.length).keys()],
y: channel,
type: 'line',
xaxis: 'x',
yaxis: `y${index + 2}`,
});
});
const subplots = [];
window.forEach((channel: number[], index: number) => {
subplots.push([`xy${index + 2}`]);
});
const plot = {
index: this.topk[idx],
data: data,
layout: {
grid: {
rows: 1, //this.state.queryWindow.length,
columns: 1,
subplots: subplots,
},
showlegend: false,
title: `Index: ${this.topk[idx].toString()}`,
hovermode: 'closest',
autosize: true,
margin: {
l: 30,
r: 30,
t: 30,
b: 0,
pad: 4
},
height: 100,
width: 300,
titlefont: {
size: 9
},
xaxis: {
showgrid: false,
zeroline: false,
showticklabels: false,
},
yaxis: {
zeroline: false,
showticklabels: false,
}
}
);
});
};
window.forEach((channel: number[], index: number) => {
plot.layout[`yaxis${index + 2}`] = {
zeroline: false,
showticklabels: false,
};
});
this.subplots.push(plot);
}
}
public get show() {
......
AngularApp/prototype/src/app/overview-window/overview-window.component.html
View file @ 6e893f4b
<zingchart-angular [id]="id" [config]="config" (mousewheel)="zoom($event)" (click)="clicked($event)" [height]="150"></zingchart-angular>
<zingchart-angular #chart [id]="id" [config]="config" (mousewheel)="zoom($event)" (click)="clicked($event)" [height]="150"></zingchart-angular>
AngularApp/prototype/src/app/overview-window/overview-window.component.ts
View file @ 6e893f4b
import {Component, OnInit} from '@angular/core';
import {Component, OnInit, ViewChild} from '@angular/core';
import { StateService } from '../state.service';
import zingchart from 'zingchart/es6';
......@@ -8,7 +8,9 @@ import zingchart from 'zingchart/es6';
styleUrls: ['./overview-window.component.css']
})
export class OverviewWindowComponent implements OnInit {
@ViewChild('chart') chart;
public config;
public graphset = [];
public id = 'overview';
public markers = [];
public series = [];
......@@ -29,61 +31,125 @@ export class OverviewWindowComponent implements OnInit {
async initializePlot() {
this.state.queryWindow = undefined;
// this.debugClicked();
this.data = [];
for (let i = 0; i < this.state.rawData.values.length; i++) {
this.data.push([this.state.rawData.index[i], this.state.rawData.values[i]]);
}
this.series = [
{
type: 'line',
values: this.data,
text: 'Raw Values',
zIndex: 5,
marker: {
alpha: 0.0,
zIndex: 10
}
},
{
type: 'scatter',
values: [],
text: 'Good labels',
marker: {
backgroundColor: '#4caf50'
},
zIndex: 20,
},
{
type: 'scatter',
values: [],
text: 'Bad labels',
marker: {
backgroundColor: '#f44336'
},
zIndex: 20,
}];
this.config = {
type: 'mixed',
preview: {
height: '30px',
position: '0% 100%',
'auto-fit': true
},
plotarea: {
'margin-top': '10px',
'margin-bottom': '50%'
},
this.graphset.push({
id: 'preview',
type: "scatter",
x: 0,
y: 0,
scaleX: {
zooming: true,
minValue: 0,
maxValue: this.state.rawData[0].values.length,
zoomTo: [0, this.state.windowSize],
'auto-fit': true,
markers: this.markers
visible: false,
guide: {
visible: false
}
},
'scale-y': {
'auto-fit': true
height: '30px',
scaleY: {
maxValue: 1,
minValue: -1,
visible: false,
guide: {
visible: false
}
},
preview: {
x: 50,
y: 10,
height: '30px',
},
series: this.series
series: [
{
type: 'scatter',
values: [],
text: 'Good labels',
marker: {
backgroundColor: '#4caf50'
},
zIndex: 3,
},
{
type: 'scatter',
values: [],
text: 'Bad labels',
marker: {
backgroundColor: '#f44336'
},
zIndex: 2,
},
{
type: 'scatter',
values: [],
text: 'Candidates',
marker: {
backgroundColor: '#b1a343'
},
zIndex: 1,
}
]
});
// Initialize channels
this.state.rawData.forEach((channel, index) => {
if (index !== 0) {
return;
}
const data = [];
for (let i = 0; i < channel.values.length; i++) {
data.push([i, channel.values[i]]);
}
this.graphset.push({
id: index,
x: 0,
y: `${75 * index + 50}px`,
type: 'line',
height: '50px',
scaleX: {
zooming: true,
zoomTo: [0, this.state.windowSize],
markers: []
},
'scale-y': {
zooming: false,
'auto-fit': true
},
plotarea: {
height: '50px',
'margin-top': '0px',
'margin-bot': '0px'
},
series: [{
type: 'line',
values: data,
text: 'Raw Values',
zIndex: 5,
marker: {
alpha: 0.0,
zIndex: 10
}
}]
});
});
zingchart.bind('zingchart-ng-1', 'zoom', (e) => {
if (e.graphid !== `zingchart-ng-1-graph-preview`) {
return;
}
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
xmin: e.xmin,
xmax: e.xmax
});
}
});
this.config = {
layout: `${this.graphset.length}x1`,
graphset: this.graphset
};
console.log(this.config);
console.log("showing plot");
}
async updatePlot() {
......@@ -170,7 +236,7 @@ export class OverviewWindowComponent implements OnInit {
}
async debugClicked() {
const index = 80503;
const index = 1234;
await this.state.getQueryWindow(index);
await this.state.lshInitial();
}
......
AngularApp/prototype/src/app/query-window/query-window.component.ts
View file @ 6e893f4b
......@@ -22,8 +22,8 @@ export class QueryWindowComponent implements OnInit {
initializePlot(): void {
const data = [{
x: [...Array(this.state.queryWindow.length).keys()],
y: this.state.queryWindow,
x: [...Array(this.state.queryWindow[0].length).keys()],
y: this.state.queryWindow[0],
type: 'line'
}];
// if (this.service.distances.length !== 0) {
......
AngularApp/prototype/src/app/state.service.ts
View file @ 6e893f4b
......@@ -7,16 +7,16 @@ import {ApiService, LshData, RawData, TableInfoData} from './api.service';
export class StateService {
public loadingProgress: number = 0;
private _rawData: RawData;
private _rawData: RawData[];
private _lshData: LshData;
private _tableInfo: TableInfoData;
private _queryWindow: number[];
private _queryWindow: number[][];
private _table: {[bucket: string]: number[]};
private _currentTab: number;
private _labels = {};
private _sliderValue;
private _parameters: number[];
private _lshParameters: number[];
public windowSize = 120;
public nrOfTables = 5;
......@@ -63,11 +63,12 @@ export class StateService {
async lshInitial(): Promise<void> {
this.lshData = await this.api.lshInitial(this._queryWindow);
this._lshParameters = this.lshData.parameters;
this.createTable();
}
async update(): Promise<void> {
this.lshData = await this.api.lshUpdate(this._queryWindow, [], this.lshData.parameters);
this.lshData = await this.api.lshUpdate(this._queryWindow, [], this._lshParameters);
this.createTable();
}
......@@ -76,12 +77,12 @@ export class StateService {
return this.tableInfo;
}
async getQueryWindow(windowIndex: number | {[index: number]: boolean}): Promise<number[]> {
async getQueryWindow(windowIndex: number | {[index: number]: boolean}): Promise<number[][]> {
this.queryWindow = await this.api.getQueryWindow(windowIndex);
return this._queryWindow;
}
async getWindow(indices: number[]): Promise<number[][]> {
async getWindow(indices: number[]): Promise<number[][][]> {
return await this.api.getWindowByIndices(indices);
}
......@@ -99,12 +100,13 @@ export class StateService {
this.getTableInfo();
}
public set rawData(v: RawData) {
public set rawData(v: RawData[]) {
this._rawData = v;
console.log(this._rawData);
this.onNewData.emit();
}
public get rawData(): RawData {
public get rawData(): RawData []{
return this._rawData;
}
......@@ -163,15 +165,19 @@ export class StateService {
return this._sliderValue;
}
public set queryWindow(v: number[]) {
public set queryWindow(v: number[][]) {
this._queryWindow = v;
this.onNewQuery.emit();
}
public get queryWindow(): number[] {
public get queryWindow(): number[][] {
return this._queryWindow;
}
public get lshParameters(): number[] {
return this._lshParameters;
}
public get parameters(): {[parameter: string]: number} {
return {
windowsize: this.windowSize,
......
Flaskserver/.idea/workspace.xml
View file @ 6e893f4b
......@@ -20,17 +20,11 @@
</component>
<component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/1.csv" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/DailyDelhiClimateTrain.csv" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
<change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/setup.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../environment.yml" beforeDir="false" afterPath="$PROJECT_DIR$/../environment.yml" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......@@ -50,7 +44,7 @@
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/../experiments/utils.py" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
</component>
......@@ -137,13 +131,21 @@
<screen x="72" y="27" width="1848" height="1053" />
</state>
<state x="686" y="355" width="777" height="403" key="#com.intellij.fileTypes.FileTypeChooser/72.27.1848.1053@72.27.1848.1053" timestamp="1603212805629" />
<state x="729" y="302" width="524" height="509" key="#com.intellij.refactoring.safeDelete.UnsafeUsagesDialog" timestamp="1605036046833">
<screen x="72" y="27" width="1848" height="1053" />
</state>
<state x="729" y="302" width="524" height="509" key="#com.intellij.refactoring.safeDelete.UnsafeUsagesDialog/72.27.1848.1053@72.27.1848.1053" timestamp="1605036046833" />
<state x="479" y="254" width="1200" height="800" key="DiffContextDialog" timestamp="1603129938934">
<screen x="72" y="27" width="1848" height="1053" />
</state>
<state x="479" y="254" width="1200" height="800" key="DiffContextDialog/72.27.1848.1053@72.27.1848.1053" timestamp="1603129938934" />
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl" timestamp="1603390700256">
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl" timestamp="1604916700277">
<screen x="72" y="27" width="1848" height="1053" />
</state>
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl/72.27.1848.1053@72.27.1848.1053" timestamp="1604916700277" />
<state x="659" y="259" width="672" height="678" key="search.everywhere.popup" timestamp="1604929652702">
<screen x="72" y="27" width="1848" height="1053" />
</state>
<state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl/72.27.1848.1053@72.27.1848.1053" timestamp="1603390700256" />
<state x="659" y="259" width="672" height="678" key="search.everywhere.popup/72.27.1848.1053@72.27.1848.1053" timestamp="1604929652702" />
</component>
</project>
\ No newline at end of file
Flaskserver/data.pkl
View file @ 6e893f4b
No preview for this file type
Flaskserver/main.py
View file @ 6e893f4b
......@@ -2,6 +2,7 @@ from flask import Flask, request
import numpy as np
from flask_cors import CORS
from time import time
import pandas as pd
import orjson
import bigwig
import bbi
......@@ -9,8 +10,12 @@ import _ucrdtw
import _lsh
import dtw
import math
import dask.dataframe as dd
import os.path
from random import sample
from DBA import performDBA
from DBA_multivariate import performDBA
from tslearn.metrics import dtw
from sklearn import preprocessing
reload = False
......@@ -36,6 +41,68 @@ def read_data():
print('Data read: ' + str(time()-t0))
return response
@app.route('/read-mts-data', methods=['GET'])
def read_mts_data():
filename = 'data.pkl'
if (not os.path.isfile(filename)):
print("start")
df = dd.read_csv("NW_Ground_Stations_2016.csv", usecols=['number_sta', 'date', 't', 'hu', 'td'])
print("read file")
df = df.loc[df['number_sta'] == 14066001]
print("split rows")
df = df.compute()
df.to_pickle(filename)
print("to_pandas")
df = pd.read_pickle(filename)
df.dropna(subset=['t'], inplace=True)
response = [
{
"index": df.loc[:, 'date'].values.astype(str).tolist(),
"values": df.loc[:, 't'].values.tolist()
},
{
"index": df.loc[:, 'date'].values.astype(str).tolist(),
"values": df.loc[:, 'hu'].values.tolist()
},
{
"index": df.loc[:, 'date'].values.astype(str).tolist(),
"values": df.loc[:, 'td'].values.tolist()
}
]
print("response ready")
response = orjson.dumps(response)
return response
@app.route('/create-mts-windows', methods=['POST'])
def create_mts_windows():
t0 = time()
if (not os.path.isfile('processed-data.npy')):
filename = 'data.pkl'
df = pd.read_pickle(filename)
channels = list()