Skip to content

GitLab

Next tuesday 25th january around 21.30 we'll be upgrading to GitLab version 14.7

Commit 5deb5c15 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Add computing time pseudo experiment

parent ced94cba
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 408 additions and 1007 deletions
AngularApp/prototype/src/app/api.service.ts
View file @ 5deb5c15
......@@ -111,6 +111,14 @@ export class ApiService {
return await response.json();
}
/**
* Calculate the LSH parameters
*/
async getParameters(): Promise<number[]> {
const response = await fetch('http://127.0.0.1:5000/get-lsh-parameters');
return await response.json();
}
/**
* Do another iteration of LSH, with weights, and return important information
*/
......
AngularApp/prototype/src/app/overview-window/overview-window.component.html
View file @ 5deb5c15
......@@ -2,7 +2,7 @@
<as-split direction="horizontal">
<as-split-area [size]="10">
<div class="channel_header">
Channels
Tracks
</div>
<div class="content">
<button (click)="setSelection()">Set</button>
......@@ -34,8 +34,8 @@
Set as query
</button>
</div>
<div class="overview">
<zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>
<div class="overview" id="overview">
<!-- <zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>-->
</div>
</as-split-area>
</as-split>
......
AngularApp/prototype/src/app/overview-window/overview-window.component.ts
View file @ 5deb5c15
......@@ -9,7 +9,7 @@ import * as d3 from 'd3';
styleUrls: ['./overview-window.component.css']
})
export class OverviewWindowComponent implements OnInit {
@ViewChild('chart') chart;
@ViewChild('overview') chart;
private initialHeight;
public config;
public graphset = [];
......@@ -193,6 +193,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true,
markers: []
},
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': {
zooming: false,
'auto-fit': true
......@@ -215,7 +224,7 @@ export class OverviewWindowComponent implements OnInit {
}]
});
});
zingchart.bind('zingchart-ng-1', 'zoom', (e) => {
zingchart.bind('overview', 'zoom', (e) => {
console.log("zoom");
if (e.stopupdate) {
return;
......@@ -223,9 +232,11 @@ export class OverviewWindowComponent implements OnInit {
this._minx = e.xmin;
this._maxx = e.xmax;
}
if (e.graphid !== `zingchart-ng-1-graph-preview`) {
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
if (e.graphid !== `overview-graph-preview`) {
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
......@@ -235,24 +246,31 @@ export class OverviewWindowComponent implements OnInit {
if (i === e.graphid) {
continue;
}
this.chart.zoomto({
graphid: i,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
});
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: e.xmin,
xmax: e.xmax,
stopupdate: true
});
}
});
this.config = {
layout: `${this.graphset.length}x1`,
graphset: this.graphset
};
this.chart.setdata({
data: this.config
});
// this.chart.setdata({
// data: this.config
// });
console.log("showing plot");
this._minx = 0;
this._maxx = this.state.rawData[0].values.length / 10;
zingchart.render({
id: 'overview',
data: this.config,
output: 'canvas'
});
// await this.debugClicked();
}
......@@ -281,6 +299,15 @@ export class OverviewWindowComponent implements OnInit {
zooming: true,
markers: []
},
plot: {
exact: false,
smartSampling: true,
sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
scrollStepMultiplier: 5,
hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
maxTrackers: 150,
},
'scale-y': {
zooming: false,
'auto-fit': true
......@@ -307,11 +334,16 @@ export class OverviewWindowComponent implements OnInit {
layout: `${this.graphset.length}x1`,
graphset: this.graphset
};
this.chart.setdata({
data: this.config
});
// this.chart.addgraph(this.graphset[4]);
console.log("showing plot");
// this.chart.setdata({
// data: this.config
// });
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
// this.chart.addgraph(this.graphset[4]);
this.minx = 0;
this.maxx = 120;
}
......@@ -362,10 +394,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels;
}
}
this.chart.setdata({
data: this.config
});
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
}
ping() {
......@@ -423,9 +456,11 @@ export class OverviewWindowComponent implements OnInit {
channel.series[2].values = this.candidateLabels;
}
}
this.chart.setdata({
data: this.config
});
zingchart.exec(
'overview',
'setdata',
{ data: this.config }
);
}
// async clicked(clickData) {
......@@ -470,11 +505,13 @@ export class OverviewWindowComponent implements OnInit {
this._minx = Math.min(this._minx + q, this.state.rawData[0].values.length - 1);
this._maxx = Math.min(this._maxx + q, this.state.rawData[0].values.length - 1);
}
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
});
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
});
}
isInSelection(i): boolean {
......@@ -513,19 +550,23 @@ export class OverviewWindowComponent implements OnInit {
set maxx(max: number) {
this._maxx = max / 10;
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
}
}
......@@ -535,19 +576,23 @@ export class OverviewWindowComponent implements OnInit {
set minx(min: number) {
this._minx = min / 10;
this.chart.zoomto({
graphid: `zingchart-ng-1-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
this.chart.zoomto({
graphid: i,
zingchart.exec(
'overview',
'zoomto', {
graphid: `overview-graph-preview`,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
for (let i = 1; i < this.graphset.length; i ++) {
zingchart.exec(
'overview',
'zoomto', {
graphid: i,
xmin: this._minx,
xmax: this._maxx,
stopupdate: true
});
}
}
......
AngularApp/prototype/src/app/state.service.ts
View file @ 5deb5c15
......@@ -92,17 +92,21 @@ export class StateService {
this.onNewWindows.emit();
}
/**
* This function performs the first iteration of LSH
*/
async lshInitial(): Promise<void> {
this._weights = Array(this._queryWindow.length).fill(1);
const t0 = new Date().getTime();
this.loadingText = 'Calculating optimal LSH parameters';
this._lshParameters = await this.api.getParameters();
this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshInitial(this._queryWindow);
this.loadingText = '';
const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.computedTime = new Date().getTime() - t0;
this._lshParameters = this.lshData.parameters;
this.loadingText = '';
this.lshData = lshData;
}
/**
......@@ -113,9 +117,10 @@ export class StateService {
this._weights = await this.api.getWeights(this._queryWindow, labels, this._weights, hashFunctions);
const t0 = new Date().getTime();
this.loadingText = 'Creating LSH model';
this.lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.loadingText = '';
const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
this.computedTime = new Date().getTime() - t0;
this.loadingText = '';
this.lshData = lshData;
}
/**
......
Flaskserver/.idea/workspace.xml
View file @ 5deb5c15
......@@ -20,16 +20,12 @@
</component>
<component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pseudo.py" beforeDir="false" afterPath="$PROJECT_DIR$/pseudo.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
......
Flaskserver/main.py
View file @ 5deb5c15
......@@ -97,6 +97,17 @@ def initialize():
print('LSH done: ' + str(time()-t0))
return response
@app.route('/get-lsh-parameters', methods=['GET'])
def get_lsh_parameters():
t0 = time()
data = np.load(data_path)
data = np.swapaxes(data, 1, 2)
parameters = pseudo.get_lsh_parameters(data)
response = orjson.dumps(parameters)
print('Parameter calculation done: ' + str(time()-t0))
return response
"""
Does LSH and returns a bunch of useful information
......
Flaskserver/pseudo.py
View file @ 5deb5c15
......@@ -7,6 +7,13 @@ from libs.DBA_multivariate import performDBA
from tslearn.metrics import dtw
from collections import defaultdict
"""
data: 3d array [m][t][d]
"""
def get_lsh_parameters(data):
parameters = preprocess(data)
return [float(parameters[1]), float(parameters[1]), float(parameters[1])]
"""
data: 3d array [m][t][d]
query: 2d array [t][d]
......
experiments/Test computing time of large time series.ipynb 0 → 100644
View file @ 5deb5c15
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"from time import time\n",
"\n",
"datafile = 'data/21.csv'\n",
"\n",
"N = 100\n",
"T = 100\n",
"M = 100000\n",
"\n",
"data = np.random.uniform(size=(M, T, N))\n",
"\n",
"#and convert it to numpy array:\n",
"data = np.array(data, dtype = \"float32\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We sample a number of subwindows which will be used as query for the search algorithms"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[23484, 81670, 8609, 31051, 42832, 14303, 85653, 67886, 17319, 93188]\n"
]
}
],
"source": [
"import random\n",
"from time import time\n",
"\n",
"targets = random.sample(list(range(len(data))), 10)\n",
"print(targets)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## PSEUDo"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"For the LSH algorithm some preprocessing is done to find the right LSH parameters."
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Preprocessing:\n",
"r = 100\n",
"smaller\n",
"r = 50.0\n",
"smaller\n",
"r = 25.0\n",
"bigger\n",
"r = 37.5\n",
"bigger\n",
"r = 43.75\n",
"smaller\n",
"r = 40.625\n",
"bigger\n",
"r = 42.1875\n",
"smaller\n",
"r = 41.40625\n",
"smaller\n",
"r = 41.015625\n",
"Mean: 41.028657241907524\n",
"Stdev: 0.16600796717920388\n",
"Ratio mean: 0.9930308931432998\n",
"Ratio stdev: 0.004133342888573115\n",
"Theta: 40.60035668658518\n",
"r: 3.7478993980502606\n",
"Preprocessing time: 11.971427917480469\n",
"Preprocessing done. Took 11.97 seconds (0.2 minutes).\n"
]
}
],
"source": [
"import sys\n",
"\n",
"sys.path.insert(0, '../Flaskserver')\n",
"import importlib\n",
"from pseudo import preprocess\n",
"import _lsh\n",
"\n",
"topk_dtw = []\n",
"\n",
"print('Preprocessing:')\n",
"t0 = time()\n",
"r,a,sd = preprocess(data, data.shape[2])\n",
"print('Preprocessing done. Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))\n",
"pseudo_preprocess_time = time() - t0"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now we run the LSH algorithm for all targets and calculate the most similar subwindows"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"doing lsh\n",
"Target #0 done! Took 14.48 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #1 done! Took 14.31 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #2 done! Took 14.30 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #3 done! Took 14.17 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #4 done! Took 14.30 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #5 done! Took 14.31 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #6 done! Took 14.18 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #7 done! Took 13.75 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #8 done! Took 13.80 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #9 done! Took 14.17 seconds (0.2 minutes).\n",
"Done! Took 141.76 seconds (2.4 minutes).\n"
]
}
],
"source": [
"from collections import defaultdict\n",
"t0 = time()\n",
"total_lsh_times = []\n",
"all_lsh_candidates = []\n",
"for i, target in enumerate(targets):\n",
" t1 = time()\n",
" query = data[target]\n",
" print('doing lsh')\n",
" lsh_candidates, lsh_distances, _ = _lsh.lsh(data, query, r, a, sd, 0)\n",
"# topk_dtw.append(candidates)\n",
" dict = defaultdict(int)\n",
" for l in range(len(lsh_candidates)):\n",
" for k in range(len(lsh_candidates[0])):\n",
" for a in range(len(lsh_candidates[0][0])):\n",
" dict[lsh_candidates[l][k][a]] += lsh_distances[l][k][a]\n",
" sorted_dict = {k: v for k, v in sorted(dict.items(), key=lambda item: item[1])}\n",
" candidates = list(sorted_dict.keys())\n",
" total_lsh_times.append(time()-t1)\n",
" print('Target #{} done! Took {:.2f} seconds ({:.1f} minutes).'.format(i, time() - t1, (time() - t1) / 60))\n",
" all_lsh_candidates.append(candidates)\n",
" \n",
"# print(candidates[0:10])\n",
"print('Done! Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"doing lsh\n",
"Target #0 done! Took 9.85 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #1 done! Took 9.73 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #2 done! Took 9.58 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #3 done! Took 9.60 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #4 done! Took 9.84 seconds (0.2 minutes).\n",
"doing lsh\n",
"Target #5 done! Took 9.74 seconds (0.2 minutes).\n",
"doing lsh\n"
]
}
],
"source": [
"from collections import defaultdict\n",
"t0 = time()\n",
"total_lsh_times_ed = []\n",
"all_lsh_candidates_ed = []\n",
"for i, target in enumerate(targets):\n",
" t1 = time()\n",
" query = data[target]\n",
" print('doing lsh')\n",
" lsh_candidates, lsh_distances, _ = _lsh.lsh(data, query, r, a, sd, 1)\n",
"# topk_dtw.append(candidates)\n",
" dict = defaultdict(int)\n",