Commit b89119be authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Add computing time pseudo experiment


Former-commit-id: 5deb5c15
parent 69d82374
...@@ -111,6 +111,14 @@ export class ApiService { ...@@ -111,6 +111,14 @@ export class ApiService {
return await response.json(); return await response.json();
} }
/**
* Calculate the LSH parameters
*/
async getParameters(): Promise<number[]> {
const response = await fetch('http://127.0.0.1:5000/get-lsh-parameters');
return await response.json();
}
/** /**
* Do another iteration of LSH, with weights, and return important information * Do another iteration of LSH, with weights, and return important information
*/ */
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<as-split direction="horizontal"> <as-split direction="horizontal">
<as-split-area [size]="10"> <as-split-area [size]="10">
<div class="channel_header"> <div class="channel_header">
Channels Tracks
</div> </div>
<div class="content"> <div class="content">
<button (click)="setSelection()">Set</button> <button (click)="setSelection()">Set</button>
...@@ -34,8 +34,8 @@ ...@@ -34,8 +34,8 @@
Set as query Set as query
</button> </button>
</div> </div>
<div class="overview"> <div class="overview" id="overview">
<zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular> <!-- <zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>-->
</div> </div>
</as-split-area> </as-split-area>
</as-split> </as-split>
......
...@@ -9,7 +9,7 @@ import * as d3 from 'd3'; ...@@ -9,7 +9,7 @@ import * as d3 from 'd3';
styleUrls: ['./overview-window.component.css'] styleUrls: ['./overview-window.component.css']
}) })
export class OverviewWindowComponent implements OnInit { export class OverviewWindowComponent implements OnInit {
@ViewChild('chart') chart; @ViewChild('overview') chart;
private initialHeight; private initialHeight;
public config; public config;
public graphset = []; public graphset = [];
...@@ -193,6 +193,15 @@ export class OverviewWindowComponent implements OnInit { ...@@ -193,6 +193,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true, zooming: true,
markers: [] markers: []
}, },
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': { 'scale-y': {
zooming: false, zooming: false,
'auto-fit': true 'auto-fit': true
...@@ -215,7 +224,7 @@ export class OverviewWindowComponent implements OnInit { ...@@ -215,7 +224,7 @@ export class OverviewWindowComponent implements OnInit {
}] }]
}); });
}); });
zingchart.bind('zingchart-ng-1', 'zoom', (e) => { zingchart.bind('overview', 'zoom', (e) => {
console.log("zoom"); console.log("zoom");
if (e.stopupdate) { if (e.stopupdate) {
return; return;
...@@ -223,9 +232,11 @@ export class OverviewWindowComponent implements OnInit { ...@@ -223,9 +232,11 @@ export class OverviewWindowComponent implements OnInit {
this._minx = e.xmin; this._minx = e.xmin;
this._maxx = e.xmax; this._maxx = e.xmax;
} }
if (e.graphid !== `zingchart-ng-1-graph-preview`) { if (e.graphid !== `overview-graph-preview`) {
this.chart.zoomto({ zingchart.exec(
graphid: `zingchart-ng-1-graph-preview`, 'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: e.xmin, xmin: e.xmin,
xmax: e.xmax, xmax: e.xmax,
stopupdate: true stopupdate: true
...@@ -235,24 +246,31 @@ export class OverviewWindowComponent implements OnInit { ...@@ -235,24 +246,31 @@ export class OverviewWindowComponent implements OnInit {
if (i === e.graphid) { if (i === e.graphid) {
continue; continue;
} }
this.chart.zoomto({ zingchart.exec(
graphid: i, 'overview',
xmin: e.xmin, 'zoomto', {
xmax: e.xmax, graphid: i,
stopupdate: true xmin: e.xmin,
}); xmax: e.xmax,
stopupdate: true
});
} }
}); });
this.config = { this.config = {
layout: `${this.graphset.length}x1`, layout: `${this.graphset.length}x1`,
graphset: this.graphset graphset: this.graphset
}; };
this.chart.setdata({ // this.chart.setdata({
data: this.config // data: this.config
}); // });
console.log("showing plot"); console.log("showing plot");
this._minx = 0; this._minx = 0;
this._maxx = this.state.rawData[0].values.length / 10; this._maxx = this.state.rawData[0].values.length / 10;
zingchart.render({
id: 'overview',
data: this.config,
output: 'canvas'
});
// await this.debugClicked(); // await this.debugClicked();
} }
...@@ -281,6 +299,15 @@ export class OverviewWindowComponent implements OnInit { ...@@ -281,6 +299,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true, zooming: true,
markers: [] markers: []
}, },
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': { 'scale-y': {
zooming: false, zooming: false,
'auto-fit': true 'auto-fit': true
...@@ -307,11 +334,16 @@ export class OverviewWindowComponent implements OnInit { ...@@ -307,11 +334,16 @@ export class OverviewWindowComponent implements OnInit {
layout: `${this.graphset.length}x1`, layout: `${this.graphset.length}x1`,
graphset: this.graphset graphset: this.graphset
}; };
this.chart.setdata({
data: this.config
});
// this.chart.addgraph(this.graphset[4]);
console.log("showing plot"); console.log("showing plot");
// this.chart.setdata({
// data: this.config
// });
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
// this.chart.addgraph(this.graphset[4]);
this.minx = 0; this.minx = 0;
this.maxx = 120; this.maxx = 120;
} }
...@@ -362,10 +394,11 @@ export class OverviewWindowComponent implements OnInit { ...@@ -362,10 +394,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels; channel.series[2].values = this.candidateLabels;
} }
} }
this.chart.setdata({ zingchart.exec(
data: this.config 'overview',
}); 'setdata',
{ data: this.config }
);
} }
ping() { ping() {
...@@ -423,9 +456,11 @@ export class OverviewWindowComponent implements OnInit { ...@@ -423,9 +456,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels; channel.series[2].values = this.candidateLabels;
} }
} }
this.chart.setdata({ zingchart.exec(
data: this.config 'overview',
}); 'setdata',
{ data: this.config }
);
} }
// async clicked(clickData) { // async clicked(clickData) {
...@@ -470,11 +505,13 @@ export class OverviewWindowComponent implements OnInit { ...@@ -470,11 +505,13 @@ export class OverviewWindowComponent implements OnInit {
this._minx = Math.min(this._minx + q, this.state.rawData[0].values.length - 1); this._minx = Math.min(this._minx + q, this.state.rawData[0].values.length - 1);
this._maxx = Math.min(this._maxx + q, this.state.rawData[0].values.length - 1); this._maxx = Math.min(this._maxx + q, this.state.rawData[0].values.length - 1);
} }
this.chart.zoomto({ zingchart.exec(
graphid: `zingchart-ng-1-graph-preview`, 'overview',
xmin: this._minx, 'zoomto', {
xmax: this._maxx, graphid: `overview-graph-preview`,
}); xmin: this._minx,
xmax: this._maxx,
});
} }
isInSelection(i): boolean { isInSelection(i): boolean {
...@@ -513,19 +550,23 @@ export class OverviewWindowComponent implements OnInit { ...@@ -513,19 +550,23 @@ export class OverviewWindowComponent implements OnInit {
set maxx(max: number) { set maxx(max: number) {
this._maxx = max / 10; this._maxx = max / 10;
this.chart.zoomto({ zingchart.exec(
graphid: `zingchart-ng-1-graph-preview`, 'overview',
xmin: this._minx, 'zoomto', {
xmax: this._maxx, graphid: `overview-graph-preview`,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
xmin: this._minx, xmin: this._minx,
xmax: this._maxx, xmax: this._maxx,
stopupdate: true stopupdate: true
}); });
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
} }
} }
...@@ -535,19 +576,23 @@ export class OverviewWindowComponent implements OnInit { ...@@ -535,19 +576,23 @@ export class OverviewWindowComponent implements OnInit {
set minx(min: number) { set minx(min: number) {
this._minx = min / 10; this._minx = min / 10;
this.chart.zoomto({ zingchart.exec(
graphid: `zingchart-ng-1-graph-preview`, 'overview',
xmin: this._minx, 'zoomto', {
xmax: this._maxx, graphid: `overview-graph-preview`,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
xmin: this._minx, xmin: this._minx,
xmax: this._maxx, xmax: this._maxx,
stopupdate: true stopupdate: true
}); });
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
} }
} }
......
...@@ -92,17 +92,21 @@ export class StateService { ...@@ -92,17 +92,21 @@ export class StateService {
this.onNewWindows.emit(); this.onNewWindows.emit();
} }
/** /**
* This function performs the first iteration of LSH * This function performs the first iteration of LSH
*/ */
async lshInitial(): Promise<void> { async lshInitial(): Promise<void> {
this._weights = Array(this._queryWindow.length).fill(1); this._weights = Array(this._queryWindow.length).fill(1);
const t0 = new Date().getTime(); const t0 = new Date().getTime();
this.loadingText = 'Calculating optimal LSH parameters';
this._lshParameters = await this.api.getParameters();
this.loadingText = 'Creating LSH model'; this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshInitial(this._queryWindow); const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.loadingText = '';
this.computedTime = new Date().getTime() - t0; this.computedTime = new Date().getTime() - t0;
this._lshParameters = this.lshData.parameters; this.loadingText = '';
this.lshData = lshData;
} }
/** /**
...@@ -113,9 +117,10 @@ export class StateService { ...@@ -113,9 +117,10 @@ export class StateService {
this._weights = await this.api.getWeights(this._queryWindow, labels, this._weights, hashFunctions); this._weights = await this.api.getWeights(this._queryWindow, labels, this._weights, hashFunctions);
const t0 = new Date().getTime(); const t0 = new Date().getTime();
this.loadingText = 'Creating LSH model'; this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters); const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.loadingText = '';
this.computedTime = new Date().getTime() - t0; this.computedTime = new Date().getTime() - t0;
this.loadingText = '';
this.lshData = lshData;
} }
/** /**
......
...@@ -20,16 +20,12 @@ ...@@ -20,16 +20,12 @@
</component> </component>
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment=""> <list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" /> <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" /> <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" /> <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" /> <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pseudo.py" beforeDir="false" afterPath="$PROJECT_DIR$/pseudo.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/pseudo.py" beforeDir="false" afterPath="$PROJECT_DIR$/pseudo.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" afterDir="false" />
</list> </list>
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" /> <option name="HIGHLIGHT_CONFLICTS" value="true" />
......
...@@ -97,6 +97,17 @@ def initialize(): ...@@ -97,6 +97,17 @@ def initialize():
print('LSH done: ' + str(time()-t0)) print('LSH done: ' + str(time()-t0))
return response return response
@app.route('/get-lsh-parameters', methods=['GET'])
def get_lsh_parameters():
t0 = time()
data = np.load(data_path)
data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data)
response = orjson.dumps(parameters)
print('Parameter calculation done: ' + str(time()-t0))
return response
""" """
Does LSH and returns a bunch of useful information Does LSH and returns a bunch of useful information
......
...@@ -7,6 +7,13 @@ from libs.DBA_multivariate import performDBA ...@@ -7,6 +7,13 @@ from libs.DBA_multivariate import performDBA
from tslearn.metrics import dtw from tslearn.metrics import dtw
from collections import defaultdict from collections import defaultdict
"""
data: 3d array [m][t][d]
"""
def get_lsh_parameters(data):
parameters = preprocess(data)
return [float(parameters[1]), float(parameters[1]), float(parameters[1])]
""" """
data: 3d array [m][t][d] data: 3d array [m][t][d]
query: 2d array [t][d] query: 2d array [t][d]
......
%% Cell type:code id: tags:
``` python
import pandas as pd
import numpy as np
from time import time
datafile = 'data/21.csv'
N = 100
T = 100
M = 100000
data = np.random.uniform(size=(M, T, N))
#and convert it to numpy array:
data = np.array(data, dtype = "float32")
```
%% Cell type:markdown id: tags:
We sample a number of subwindows which will be used as query for the search algorithms
%% Cell type:code id: tags:
``` python
import random
from time import time
targets = random.sample(list(range(len(data))), 10)
print(targets)
```
%%%% Output: stream
[23484, 81670, 8609, 31051, 42832, 14303, 85653, 67886, 17319, 93188]
%% Cell type:markdown id: tags:
## PSEUDo
%% Cell type:markdown id: tags:
For the LSH algorithm some preprocessing is done to find the right LSH parameters.
%% Cell type:code id: tags:
``` python
import sys
sys.path.insert(0, '../Flaskserver')
import importlib
from pseudo import preprocess
import _lsh
topk_dtw = []
print('Preprocessing:')
t0 = time()
r,a,sd = preprocess(data, data.shape[2])
print('Preprocessing done. Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))
pseudo_preprocess_time = time() - t0
```
%%%% Output: stream
Preprocessing:
r = 100
smaller
r = 50.0
smaller
r = 25.0
bigger
r = 37.5
bigger
r = 43.75
smaller
r = 40.625
bigger
r = 42.1875
smaller
r = 41.40625
smaller
r = 41.015625
Mean: 41.028657241907524
Stdev: 0.16600796717920388
Ratio mean: 0.9930308931432998
Ratio stdev: 0.004133342888573115
Theta: 40.60035668658518
r: 3.7478993980502606
Preprocessing time: 11.971427917480469
Preprocessing done. Took 11.97 seconds (0.2 minutes).
%% Cell type:markdown id: tags:
Now we run the LSH algorithm for all targets and calculate the most similar subwindows