Skip to content

GitLab

Commit 5deb5c15 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Add computing time pseudo experiment

parent ced94cba
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 408 additions and 1007 deletions
AngularApp/prototype/src/app/api.service.ts
View file @ 5deb5c15
......@@ -111,6 +111,14 @@ export class ApiService {
return await response.json();
}
/**
* Calculate the LSH parameters
*/
async getParameters(): Promise<number[]> {
const response = await fetch('http://127.0.0.1:5000/get-lsh-parameters');
return await response.json();
}
/**
* Do another iteration of LSH, with weights, and return important information
*/
......
AngularApp/prototype/src/app/overview-window/overview-window.component.html
View file @ 5deb5c15
......@@ -2,7 +2,7 @@
<as-split direction="horizontal">
<as-split-area [size]="10">
<div class="channel_header">
Channels
Tracks
</div>
<div class="content">
<button (click)="setSelection()">Set</button>
......@@ -34,8 +34,8 @@
Set as query
</button>
</div>
<div class="overview">
<zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>
<div class="overview" id="overview">
<!-- <zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>-->
</div>
</as-split-area>
</as-split>
......
AngularApp/prototype/src/app/overview-window/overview-window.component.ts
View file @ 5deb5c15
......@@ -9,7 +9,7 @@ import * as d3 from 'd3';
styleUrls: ['./overview-window.component.css']
})
export class OverviewWindowComponent implements OnInit {
@ViewChild('chart') chart;
@ViewChild('overview') chart;
private initialHeight;
public config;
public graphset = [];
......@@ -193,6 +193,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true,
markers: []
},
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': {
zooming: false,
'auto-fit': true
......@@ -215,7 +224,7 @@ export class OverviewWindowComponent implements OnInit {
}]
});
});
zingchart.bind('zingchart-ng-1', 'zoom', (e) => {
zingchart.bind('overview', 'zoom', (e) => {
console.log("zoom");
if (e.stopupdate) {
return;
......@@ -223,9 +232,11 @@ export class OverviewWindowComponent implements OnInit {
this._minx = e.xmin;
this._maxx = e.xmax;
}
if (e.graphid !== `zingchart-ng-1-graph-preview`) {
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
if (e.graphid !== `overview-graph-preview`) {
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
......@@ -235,24 +246,31 @@ export class OverviewWindowComponent implements OnInit {
if (i === e.graphid) {
continue;
}
this.chart.zoomto({
graphid: i,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
});
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
});
}
});
this.config = {
layout: `${this.graphset.length}x1`,
graphset: this.graphset
};
this.chart.setdata({
data: this.config
});
// this.chart.setdata({
// data: this.config
// });
console.log("showing plot");
this._minx = 0;
this._maxx = this.state.rawData[0].values.length / 10;
zingchart.render({
id: 'overview',
data: this.config,
output: 'canvas'
});
// await this.debugClicked();
}
......@@ -281,6 +299,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true,
markers: []
},
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': {
zooming: false,
'auto-fit': true
......@@ -307,11 +334,16 @@ export class OverviewWindowComponent implements OnInit {
layout: `${this.graphset.length}x1`,
graphset: this.graphset
};
this.chart.setdata({
data: this.config
});
// this.chart.addgraph(this.graphset[4]);
console.log("showing plot");
// this.chart.setdata({
// data: this.config
// });
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
// this.chart.addgraph(this.graphset[4]);
this.minx = 0;
this.maxx = 120;
}
......@@ -362,10 +394,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels;
}
}
this.chart.setdata({
data: this.config
});
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
}
ping() {
......@@ -423,9 +456,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels;
}
}
this.chart.setdata({
data: this.config
});
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
}
// async clicked(clickData) {
......@@ -470,11 +505,13 @@ export class OverviewWindowComponent implements OnInit {
this._minx = Math.min(this._minx + q, this.state.rawData[0].values.length - 1);
this._maxx = Math.min(this._maxx + q, this.state.rawData[0].values.length - 1);
}
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
});
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
});
}
isInSelection(i): boolean {
......@@ -513,19 +550,23 @@ export class OverviewWindowComponent implements OnInit {
set maxx(max: number) {
this._maxx = max / 10;
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
}
}
......@@ -535,19 +576,23 @@ export class OverviewWindowComponent implements OnInit {
set minx(min: number) {
this._minx = min / 10;
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
}
}
......
AngularApp/prototype/src/app/state.service.ts
View file @ 5deb5c15
......@@ -92,17 +92,21 @@ export class StateService {
this.onNewWindows.emit();
}
/**
* This function performs the first iteration of LSH
*/
async lshInitial(): Promise<void> {
this._weights = Array(this._queryWindow.length).fill(1);
const t0 = new Date().getTime();
this.loadingText = 'Calculating optimal LSH parameters';
this._lshParameters = await this.api.getParameters();
this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshInitial(this._queryWindow);
this.loadingText = '';
const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.computedTime = new Date().getTime() - t0;
this._lshParameters = this.lshData.parameters;
this.loadingText = '';
this.lshData = lshData;
}
/**
......@@ -113,9 +117,10 @@ export class StateService {
this._weights = await this.api.getWeights(this._queryWindow, labels, this._weights, hashFunctions);
const t0 = new Date().getTime();
this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.loadingText = '';
const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.computedTime = new Date().getTime() - t0;
this.loadingText = '';
this.lshData = lshData;
}
/**
......
Flaskserver/.idea/workspace.xml
View file @ 5deb5c15
......@@ -20,16 +20,12 @@
</component>
<component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pseudo.py" beforeDir="false" afterPath="$PROJECT_DIR$/pseudo.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......
Flaskserver/main.py
View file @ 5deb5c15
......@@ -97,6 +97,17 @@ def initialize():
print('LSH done: ' + str(time()-t0))
return response
@app.route('/get-lsh-parameters', methods=['GET'])
def get_lsh_parameters():
t0 = time()
data = np.load(data_path)
data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data)
response = orjson.dumps(parameters)
print('Parameter calculation done: ' + str(time()-t0))
return response
"""
Does LSH and returns a bunch of useful information
......
Flaskserver/pseudo.py
View file @ 5deb5c15
......@@ -7,6 +7,13 @@ from libs.DBA_multivariate import performDBA
from tslearn.metrics import dtw
from collections import defaultdict
"""
data: 3d array [m][t][d]
"""
def get_lsh_parameters(data):
parameters = preprocess(data)
return [float(parameters[1]), float(parameters[1]), float(parameters[1])]
"""
data: 3d array [m][t][d]
query: 2d array [t][d]
......
experiments/Test computing time of large time series.ipynb 0 → 100644
View file @ 5deb5c15
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"from time import time\n",
"\n",
"datafile = 'data/21.csv'\n",
"\n",
"N = 100\n",
"T = 100\n",
"M = 100000\n",
"\n",
"data = np.random.uniform(size=(M, T, N))\n",
"\n",
"#and convert it to numpy array:\n",
"data = np.array(data, dtype = \"float32\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We sample a number of subwindows which will be used as query for the search algorithms"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[23484, 81670, 8609, 31051, 42832, 14303, 85653, 67886, 17319, 93188]\n"
]
}
],
"source": [
"import random\n",
"from time import time\n",
"\n",
"targets = random.sample(list(range(len(data))), 10)\n",
"print(targets)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## PSEUDo"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For the LSH algorithm some preprocessing is done to find the right LSH parameters."
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Preprocessing:\n",
"r = 100\n",
"smaller\n",
"r = 50.0\n",
"smaller\n",
"r = 25.0\n",
"bigger\n",
"r = 37.5\n",
"bigger\n",
"r = 43.75\n",
"smaller\n",
"r = 40.625\n",
"bigger\n",
"r = 42.1875\n",
"smaller\n",
"r = 41.40625\n",
"smaller\n",
"r = 41.015625\n",
"Mean: 41.028657241907524\n",
"Stdev: 0.16600796717920388\n",
"Ratio mean: 0.9930308931432998\n",
"Ratio stdev: 0.004133342888573115\n",
"Theta: 40.60035668658518\n",
"r: 3.7478993980502606\n",
"Preprocessing time: 11.971427917480469\n",
"Preprocessing done. Took 11.97 seconds (0.2 minutes).\n"
]
}
],
"source": [
"import sys\n",
"\n",
"sys.path.insert(0, '../Flaskserver')\n",
"import importlib\n",
"from pseudo import preprocess\n",
"import _lsh\n",
"\n",
"topk_dtw = []\n",
"\n",
"print('Preprocessing:')\n",
"t0 = time()\n",
"r,a,sd = preprocess(data, data.shape[2])\n",
"print('Preprocessing done. Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))\n",
"pseudo_preprocess_time = time() - t0"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now we run the LSH algorithm for all targets and calculate the most similar subwindows"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"doing lsh\n",
"Target #0 done! Took 14.48 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #1 done! Took 14.31 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #2 done! Took 14.30 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #3 done! Took 14.17 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #4 done! Took 14.30 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #5 done! Took 14.31 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #6 done! Took 14.18 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #7 done! Took 13.75 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #8 done! Took 13.80 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #9 done! Took 14.17 seconds (0.2 minutes).\n",
"Done! Took 141.76 seconds (2.4 minutes).\n"
]
}
],
"source": [
"from collections import defaultdict\n",
"t0 = time()\n",
"total_lsh_times = []\n",
"all_lsh_candidates = []\n",
"for i, target in enumerate(targets):\n",
" t1 = time()\n",
" query = data[target]\n",
" print('doing lsh')\n",
" lsh_candidates, lsh_distances, _ = _lsh.lsh(data, query, r, a, sd, 0)\n",
"# topk_dtw.append(candidates)\n",
" dict = defaultdict(int)\n",
" for l in range(len(lsh_candidates)):\n",
" for k in range(len(lsh_candidates[0])):\n",
" for a in range(len(lsh_candidates[0][0])):\n",
" dict[lsh_candidates[l][k][a]] += lsh_distances[l][k][a]\n",
" sorted_dict = {k: v for k, v in sorted(dict.items(), key=lambda item: item[1])}\n",
" candidates = list(sorted_dict.keys())\n",
" total_lsh_times.append(time()-t1)\n",
" print('Target #{} done! Took {:.2f} seconds ({:.1f} minutes).'.format(i, time() - t1, (time() - t1) / 60))\n",
" all_lsh_candidates.append(candidates)\n",
" \n",
"# print(candidates[0:10])\n",
"print('Done! Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"doing lsh\n",
"Target #0 done! Took 9.85 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #1 done! Took 9.73 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #2 done! Took 9.58 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #3 done! Took 9.60 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #4 done! Took 9.84 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #5 done! Took 9.74 seconds (0.2 minutes).\n",
"doing lsh\n"
]
}
],
"source": [
"from collections import defaultdict\n",
"t0 = time()\n",
"total_lsh_times_ed = []\n",
"all_lsh_candidates_ed = []\n",
"for i, target in enumerate(targets):\n",
" t1 = time()\n",
" query = data[target]\n",
" print('doing lsh')\n",
" lsh_candidates, lsh_distances, _ = _lsh.lsh(data, query, r, a, sd, 1)\n",
"# topk_dtw.append(candidates)\n",
" dict = defaultdict(int)\n",
" for l in range(len(lsh_candidates)):\n",