Commit 5deb5c15 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Add computing time pseudo experiment

parent ced94cba
......@@ -111,6 +111,14 @@ export class ApiService {
return await response.json();
}
/**
* Calculate the LSH parameters
*/
async getParameters(): Promise<number[]> {
const response = await fetch('http://127.0.0.1:5000/get-lsh-parameters');
return await response.json();
}
/**
* Do another iteration of LSH, with weights, and return important information
*/
......
......@@ -2,7 +2,7 @@
<as-split direction="horizontal">
<as-split-area [size]="10">
<div class="channel_header">
Channels
Tracks
</div>
<div class="content">
<button (click)="setSelection()">Set</button>
......@@ -34,8 +34,8 @@
Set as query
</button>
</div>
<div class="overview">
<zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>
<div class="overview" id="overview">
<!-- <zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>-->
</div>
</as-split-area>
</as-split>
......
......@@ -9,7 +9,7 @@ import * as d3 from 'd3';
styleUrls: ['./overview-window.component.css']
})
export class OverviewWindowComponent implements OnInit {
@ViewChild('chart') chart;
@ViewChild('overview') chart;
private initialHeight;
public config;
public graphset = [];
......@@ -193,6 +193,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true,
markers: []
},
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': {
zooming: false,
'auto-fit': true
......@@ -215,7 +224,7 @@ export class OverviewWindowComponent implements OnInit {
}]
});
});
zingchart.bind('zingchart-ng-1', 'zoom', (e) => {
zingchart.bind('overview', 'zoom', (e) => {
console.log("zoom");
if (e.stopupdate) {
return;
......@@ -223,9 +232,11 @@ export class OverviewWindowComponent implements OnInit {
this._minx = e.xmin;
this._maxx = e.xmax;
}
if (e.graphid !== `zingchart-ng-1-graph-preview`) {
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
if (e.graphid !== `overview-graph-preview`) {
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
......@@ -235,24 +246,31 @@ export class OverviewWindowComponent implements OnInit {
if (i === e.graphid) {
continue;
}
this.chart.zoomto({
graphid: i,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
});
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
});
}
});
this.config = {
layout: `${this.graphset.length}x1`,
graphset: this.graphset
};
this.chart.setdata({
data: this.config
});
// this.chart.setdata({
// data: this.config
// });
console.log("showing plot");
this._minx = 0;
this._maxx = this.state.rawData[0].values.length / 10;
zingchart.render({
id: 'overview',
data: this.config,
output: 'canvas'
});
// await this.debugClicked();
}
......@@ -281,6 +299,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true,
markers: []
},
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': {
zooming: false,
'auto-fit': true
......@@ -307,11 +334,16 @@ export class OverviewWindowComponent implements OnInit {
layout: `${this.graphset.length}x1`,
graphset: this.graphset
};
this.chart.setdata({
data: this.config
});
// this.chart.addgraph(this.graphset[4]);
console.log("showing plot");
// this.chart.setdata({
// data: this.config
// });
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
// this.chart.addgraph(this.graphset[4]);
this.minx = 0;
this.maxx = 120;
}
......@@ -362,10 +394,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels;
}
}
this.chart.setdata({
data: this.config
});
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
}
ping() {
......@@ -423,9 +456,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels;
}
}
this.chart.setdata({
data: this.config
});
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
}
// async clicked(clickData) {
......@@ -470,11 +505,13 @@ export class OverviewWindowComponent implements OnInit {
this._minx = Math.min(this._minx + q, this.state.rawData[0].values.length - 1);
this._maxx = Math.min(this._maxx + q, this.state.rawData[0].values.length - 1);
}
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
});
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
});
}
isInSelection(i): boolean {
......@@ -513,19 +550,23 @@ export class OverviewWindowComponent implements OnInit {
set maxx(max: number) {
this._maxx = max / 10;
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
}
}
......@@ -535,19 +576,23 @@ export class OverviewWindowComponent implements OnInit {
set minx(min: number) {
this._minx = min / 10;
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
}
}
......
......@@ -92,17 +92,21 @@ export class StateService {
this.onNewWindows.emit();
}
/**
* This function performs the first iteration of LSH
*/
async lshInitial(): Promise<void> {
this._weights = Array(this._queryWindow.length).fill(1);
const t0 = new Date().getTime();
this.loadingText = 'Calculating optimal LSH parameters';
this._lshParameters = await this.api.getParameters();
this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshInitial(this._queryWindow);
this.loadingText = '';
const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.computedTime = new Date().getTime() - t0;
this._lshParameters = this.lshData.parameters;
this.loadingText = '';
this.lshData = lshData;
}
/**
......@@ -113,9 +117,10 @@ export class StateService {
this._weights = await this.api.getWeights(this._queryWindow, labels, this._weights, hashFunctions);
const t0 = new Date().getTime();
this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.loadingText = '';
const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.computedTime = new Date().getTime() - t0;
this.loadingText = '';
this.lshData = lshData;
}
/**
......
......@@ -20,16 +20,12 @@
</component>
<component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pseudo.py" beforeDir="false" afterPath="$PROJECT_DIR$/pseudo.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......
......@@ -97,6 +97,17 @@ def initialize():
print('LSH done: ' + str(time()-t0))
return response
@app.route('/get-lsh-parameters', methods=['GET'])
def get_lsh_parameters():
t0 = time()
data = np.load(data_path)
data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data)
response = orjson.dumps(parameters)
print('Parameter calculation done: ' + str(time()-t0))
return response
"""
Does LSH and returns a bunch of useful information
......
......@@ -7,6 +7,13 @@ from libs.DBA_multivariate import performDBA
from tslearn.metrics import dtw
from collections import defaultdict
"""
data: 3d array [m][t][d]
"""
def get_lsh_parameters(data):
parameters = preprocess(data)
return [float(parameters[1]), float(parameters[1]), float(parameters[1])]
"""
data: 3d array [m][t][d]
query: 2d array [t][d]
......
%% Cell type:code id: tags:
``` python
import pandas as pd
import numpy as np
from time import time
datafile = 'data/21.csv'
N = 100
T = 100
M = 100000
data = np.random.uniform(size=(M, T, N))
#and convert it to numpy array:
data = np.array(data, dtype = "float32")
```
%% Cell type:markdown id: tags:
We sample a number of subwindows which will be used as query for the search algorithms
%% Cell type:code id: tags:
``` python
import random
from time import time
targets = random.sample(list(range(len(data))), 10)
print(targets)
```
%%%% Output: stream
[23484, 81670, 8609, 31051, 42832, 14303, 85653, 67886, 17319, 93188]
%% Cell type:markdown id: tags:
## PSEUDo
%% Cell type:markdown id: tags:
For the LSH algorithm some preprocessing is done to find the right LSH parameters.
%% Cell type:code id: tags:
``` python
import sys
sys.path.insert(0, '../Flaskserver')
import importlib
from pseudo import preprocess
import _lsh
topk_dtw = []
print('Preprocessing:')
t0 = time()
r,a,sd = preprocess(data, data.shape[2])
print('Preprocessing done. Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))
pseudo_preprocess_time = time() - t0
```
%%%% Output: stream
Preprocessing:
r = 100
smaller
r = 50.0
smaller
r = 25.0
bigger
r = 37.5
bigger
r = 43.75
smaller
r = 40.625
bigger
r = 42.1875
smaller
r = 41.40625
smaller
r = 41.015625
Mean: 41.028657241907524
Stdev: 0.16600796717920388
Ratio mean: 0.9930308931432998
Ratio stdev: 0.004133342888573115
Theta: 40.60035668658518
r: 3.7478993980502606
Preprocessing time: 11.971427917480469
Preprocessing done. Took 11.97 seconds (0.2 minutes).
%% Cell type:markdown id: tags:
Now we run the LSH algorithm for all targets and calculate the most similar subwindows
%% Cell type:code id: tags:
``` python
from collections import defaultdict
t0 = time()
total_lsh_times = []
all_lsh_candidates = []
for i, target in enumerate(targets):
t1 = time()
query = data[target]
print('doing lsh')
lsh_candidates, lsh_distances, _ = _lsh.lsh(data, query, r, a, sd, 0)
# topk_dtw.append(candidates)
dict = defaultdict(int)
for l in range(len(lsh_candidates)):
for k in range(len(lsh_candidates[0])):
for a in range(len(lsh_candidates[0][0])):
dict[lsh_candidates[l][k][a]] += lsh_distances[l][k][a]
sorted_dict = {k: v for k, v in sorted(dict.items(), key=lambda item: item[1])}
candidates = list(sorted_dict.keys())
total_lsh_times.append(time()-t1)
print('Target #{} done! Took {:.2f} seconds ({:.1f} minutes).'.format(i, time() - t1, (time() - t1) / 60))
all_lsh_candidates.append(candidates)
# print(candidates[0:10])
print('Done! Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))
```
%%%% Output: stream
doing lsh
Target #0 done! Took 14.48 seconds (0.2 minutes).
doing lsh
Target #1 done! Took 14.31 seconds (0.2 minutes).
doing lsh
Target #2 done! Took 14.30 seconds (0.2 minutes).
doing lsh
Target #3 done! Took 14.17 seconds (0.2 minutes).
doing lsh
Target #4 done! Took 14.30 seconds (0.2 minutes).
doing lsh
Target #5 done! Took 14.31 seconds (0.2 minutes).
doing lsh
Target #6 done! Took 14.18 seconds (0.2 minutes).
doing lsh
Target #7 done! Took 13.75 seconds (0.2 minutes).
doing lsh
Target #8 done! Took 13.80 seconds (0.2 minutes).
doing lsh
Target #9 done! Took 14.17 seconds (0.2 minutes).
Done! Took 141.76 seconds (2.4 minutes).
%% Cell type:code id: tags:
``` python
from collections import defaultdict
t0 = time()
total_lsh_times_ed = []
all_lsh_candidates_ed = []
for i, target in enumerate(targets):
t1 = time()
query = data[target]
print('doing lsh')
lsh_candidates, lsh_distances, _ = _lsh.lsh(data, query, r, a, sd, 1)
# topk_dtw.append(candidates)
dict = defaultdict(int)
for l in range(len(lsh_candidates)):
for k in range(len(lsh_candidates[0])):
for a in range(len(lsh_candidates[0][0])):
dict[lsh_candidates[l][k][a]] += lsh_distances[l][k][a]
sorted_dict = {k: v for k, v in sorted(dict.items(), key=lambda item: item[1])}
candidates = list(sorted_dict.keys())
total_lsh_times_ed.append(time()-t1)
print('Target #{} done! Took {:.2f} seconds ({:.1f} minutes).'.format(i, time() - t1, (time() - t1) / 60))
all_lsh_candidates_ed.append(candidates)
# print(candidates[0:10])
print('Done! Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))
```
%%%% Output: stream
doing lsh
Target #0 done! Took 9.85 seconds (0.2 minutes).
doing lsh
Target #1 done! Took 9.73 seconds (0.2 minutes).
doing lsh
Target #2 done! Took 9.58 seconds (0.2 minutes).
doing lsh
Target #3 done! Took 9.60 seconds (0.2 minutes).
doing lsh
Target #4 done! Took 9.84 seconds (0.2 minutes).
doing lsh
Target #5 done! Took 9.74 seconds (0.2 minutes).
doing lsh
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
```
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment