Add computing time pseudo experiment

Former-commit-id: 5deb5c15

AngularApp/prototype/src/app/api.service.ts

@@ -111,6 +111,14 @@ export class ApiService {
     return await response.json();
   }
 
+  /**
+   * Calculate the LSH parameters
+   */
+  async getParameters(): Promise<number[]> {
+    const response = await fetch('http://127.0.0.1:5000/get-lsh-parameters');
+    return await response.json();
+  }
+
   /**
    * Do another iteration of LSH, with weights, and return important information
    */

AngularApp/prototype/src/app/overview-window/overview-window.component.html

@@ -2,7 +2,7 @@
   <as-split direction="horizontal">
     <as-split-area [size]="10">
       <div class="channel_header">
-        Channels
+        Tracks
       </div>
       <div class="content">
         <button (click)="setSelection()">Set</button>
@@ -34,8 +34,8 @@
           Set as query
         </button>
       </div>
-      <div class="overview">
-        <zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>
+      <div class="overview" id="overview">
+<!--        <zingchart-angular #chart [id]="id" (wheel)="scroll($event)" [height]="height"></zingchart-angular>-->
       </div>
     </as-split-area>
   </as-split>

AngularApp/prototype/src/app/overview-window/overview-window.component.ts

@@ -9,7 +9,7 @@ import * as d3 from 'd3';
   styleUrls: ['./overview-window.component.css']
 })
 export class OverviewWindowComponent implements OnInit {
-  @ViewChild('chart') chart;
+  @ViewChild('overview') chart;
   private initialHeight;
   public config;
   public graphset = [];
@@ -193,6 +193,15 @@ export class OverviewWindowComponent implements OnInit {
           zooming: true,
           markers: []
         },
+        plot: {
+          exact: false,
+          smartSampling: true,
+          sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
+          scrollStepMultiplier: 5,
+          hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
+          maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
+          maxTrackers: 150,
+        },
         'scale-y': {
           zooming: false,
           'auto-fit': true
@@ -215,7 +224,7 @@ export class OverviewWindowComponent implements OnInit {
         }]
       });
     });
-    zingchart.bind('zingchart-ng-1', 'zoom', (e) => {
+    zingchart.bind('overview', 'zoom', (e) => {
       console.log("zoom");
       if (e.stopupdate) {
         return;
@@ -223,9 +232,11 @@ export class OverviewWindowComponent implements OnInit {
         this._minx = e.xmin;
         this._maxx = e.xmax;
       }
-      if (e.graphid !== `zingchart-ng-1-graph-preview`) {
-        this.chart.zoomto({
-          graphid: `zingchart-ng-1-graph-preview`,
+      if (e.graphid !== `overview-graph-preview`) {
+        zingchart.exec(
+          'overview',
+          'zoomto', {
+          graphid: `overview-graph-preview`,
           xmin: e.xmin,
           xmax: e.xmax,
           stopupdate: true
@@ -235,7 +246,9 @@ export class OverviewWindowComponent implements OnInit {
         if (i === e.graphid) {
           continue;
         }
-        this.chart.zoomto({
+        zingchart.exec(
+          'overview',
+          'zoomto', {
             graphid: i,
             xmin: e.xmin,
             xmax: e.xmax,
@@ -247,12 +260,17 @@ export class OverviewWindowComponent implements OnInit {
       layout: `${this.graphset.length}x1`,
       graphset: this.graphset
     };
-    this.chart.setdata({
-      data: this.config
-    });
+    // this.chart.setdata({
+    //   data: this.config
+    // });
     console.log("showing plot");
     this._minx = 0;
     this._maxx = this.state.rawData[0].values.length / 10;
+    zingchart.render({
+      id: 'overview',
+      data: this.config,
+      output: 'canvas'
+    });
     // await this.debugClicked();
   }
 
@@ -281,6 +299,15 @@ export class OverviewWindowComponent implements OnInit {
           zooming: true,
           markers: []
         },
+        plot: {
+          exact: false,
+          smartSampling: true,
+          sampleStep: 1, // scroll step 1 is not sampling. Setting this this value to 2 will sample every other data point
+          scrollStepMultiplier: 5,
+          hintTs: true, // tells the library you have timestamps as keys and activates a small optimization technique
+          maxNodes: 150, // max nodes to have event listeners for eg) tooltips wont show but crosshair will
+          maxTrackers: 150,
+        },
         'scale-y': {
           zooming: false,
           'auto-fit': true
@@ -307,11 +334,16 @@ export class OverviewWindowComponent implements OnInit {
       layout: `${this.graphset.length}x1`,
       graphset: this.graphset
     };
-    this.chart.setdata({
-      data: this.config
-    });
-    // this.chart.addgraph(this.graphset[4]);
     console.log("showing plot");
+    // this.chart.setdata({
+    //   data: this.config
+    // });
+    zingchart.exec(
+      'overview',
+      'setdata',
+      { data: this.config }
+    );
+    // this.chart.addgraph(this.graphset[4]);
     this.minx = 0;
     this.maxx = 120;
   }
@@ -362,10 +394,11 @@ export class OverviewWindowComponent implements OnInit {
         channel.series[2].values = this.candidateLabels;
       }
     }
-    this.chart.setdata({
-      data: this.config
-    });
-
+    zingchart.exec(
+      'overview',
+      'setdata',
+      { data: this.config }
+    );
   }
 
   ping() {
@@ -423,9 +456,11 @@ export class OverviewWindowComponent implements OnInit {
         channel.series[2].values = this.candidateLabels;
       }
     }
-    this.chart.setdata({
-      data: this.config
-    });
+    zingchart.exec(
+      'overview',
+      'setdata',
+      { data: this.config }
+    );
   }
 
   // async clicked(clickData) {
@@ -470,8 +505,10 @@ export class OverviewWindowComponent implements OnInit {
       this._minx = Math.min(this._minx + q, this.state.rawData[0].values.length - 1);
       this._maxx = Math.min(this._maxx + q, this.state.rawData[0].values.length - 1);
     }
-    this.chart.zoomto({
-      graphid: `zingchart-ng-1-graph-preview`,
+    zingchart.exec(
+      'overview',
+      'zoomto', {
+        graphid: `overview-graph-preview`,
         xmin: this._minx,
         xmax: this._maxx,
       });
@@ -513,14 +550,18 @@ export class OverviewWindowComponent implements OnInit {
 
   set maxx(max: number) {
     this._maxx = max / 10;
-    this.chart.zoomto({
-      graphid: `zingchart-ng-1-graph-preview`,
+    zingchart.exec(
+      'overview',
+      'zoomto', {
+        graphid: `overview-graph-preview`,
         xmin: this._minx,
         xmax: this._maxx,
         stopupdate: true
       });
     for (let i = 1; i < this.graphset.length; i ++) {
-      this.chart.zoomto({
+      zingchart.exec(
+        'overview',
+        'zoomto', {
           graphid: i,
           xmin: this._minx,
           xmax: this._maxx,
@@ -535,14 +576,18 @@ export class OverviewWindowComponent implements OnInit {
 
   set minx(min: number) {
     this._minx = min / 10;
-    this.chart.zoomto({
-      graphid: `zingchart-ng-1-graph-preview`,
+    zingchart.exec(
+      'overview',
+      'zoomto', {
+        graphid: `overview-graph-preview`,
         xmin: this._minx,
         xmax: this._maxx,
         stopupdate: true
       });
     for (let i = 1; i < this.graphset.length; i ++) {
-      this.chart.zoomto({
+      zingchart.exec(
+        'overview',
+        'zoomto', {
           graphid: i,
           xmin: this._minx,
           xmax: this._maxx,

AngularApp/prototype/src/app/state.service.ts

@@ -92,17 +92,21 @@ export class StateService {
     this.onNewWindows.emit();
   }
 
+
+
   /**
    * This function performs the first iteration of LSH
    */
   async lshInitial(): Promise<void> {
     this._weights = Array(this._queryWindow.length).fill(1);
     const t0 = new Date().getTime();
+    this.loadingText = 'Calculating optimal LSH parameters';
+    this._lshParameters = await this.api.getParameters();
     this.loadingText = 'Creating LSH model';
-    this.lshData = await this.api.lshInitial(this._queryWindow);
-    this.loadingText = '';
+    const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
     this.computedTime = new Date().getTime() - t0;
-    this._lshParameters = this.lshData.parameters;
+    this.loadingText = '';
+    this.lshData = lshData;
   }
 
   /**
@@ -113,9 +117,10 @@ export class StateService {
     this._weights = await this.api.getWeights(this._queryWindow, labels, this._weights, hashFunctions);
     const t0 = new Date().getTime();
     this.loadingText = 'Creating LSH model';
-    this.lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
-    this.loadingText = '';
+    const lshData = await this.api.lshUpdate(this._queryWindow, this._weights, this._lshParameters);
     this.computedTime = new Date().getTime() - t0;
+    this.loadingText = '';
+    this.lshData = lshData;
   }
 
   /**

Flaskserver/.idea/workspace.xml

@@ -20,16 +20,12 @@
   </component>
   <component name="ChangeListManager">
     <list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/app.module.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.css" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/pseudo.py" beforeDir="false" afterPath="$PROJECT_DIR$/pseudo.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/Results &amp; Graphs.ipynb" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />

Flaskserver/main.py

@@ -97,6 +97,17 @@ def initialize():
     print('LSH done: ' + str(time()-t0))
     return response
 
+@app.route('/get-lsh-parameters', methods=['GET'])
+def get_lsh_parameters():
+    t0 = time()
+    data = np.load(data_path)
+    data = np.swapaxes(data, 1, 2)
+    parameters = pseudo.get_lsh_parameters(data)
+
+    response = orjson.dumps(parameters)
+    print('Parameter calculation done: ' + str(time()-t0))
+    return response
+
 
 """
 Does LSH and returns a bunch of useful information

Flaskserver/pseudo.py

@@ -7,6 +7,13 @@ from libs.DBA_multivariate import performDBA
 from tslearn.metrics import dtw
 from collections import defaultdict
 
+"""
+data: 3d array [m][t][d]
+"""
+def get_lsh_parameters(data):
+    parameters = preprocess(data)
+    return [float(parameters[1]), float(parameters[1]), float(parameters[1])]
+
 """
 data: 3d array [m][t][d]
 query: 2d array [t][d]

experiments/Compare Recall and Precision on EEG data, M=100.000, T=128, d=35-Copy1.ipynb

-%% Cell type:markdown id: tags:
-
-# PSEUDo vs. DTW: EEG Data
-
-%% Cell type:markdown id: tags:
-
-In this experiment we will compare the LSH algorithm of PSEUDo to DTW using an EEG dataset. The metrics we will be comparing these two algorithms with are **computing time**, **recall** and **precision**.
-
-%% Cell type:markdown id: tags:
-
-We first load the EEG data and convert it to a numpy array
-
-%% Cell type:code id: tags:
-
-``` python
-import pandas as pd
-import numpy as np
-from time import time
-
-datafile = 'data/21.csv'
-
-N = 100
-T = 100
-M = 100000
-
-data = np.random.uniform(size=(M, T, N))
-
-#and convert it to numpy array:
-data = np.array(data, dtype = "float32")
-```
-
-%% Cell type:code id: tags:
-
-``` python
-from sklearn import preprocessing
-
-
-```
-
-%% Cell type:markdown id: tags:
-
-We sample a number of subwindows which will be used as query for the search algorithms
-
-%% Cell type:code id: tags:
-
-``` python
-import random
-from time import time
-
-targets = random.sample(list(range(len(data))), 10)
-print(targets)
-```
-
-%%%% Output: stream
-
-    [22515, 64560, 89962, 80440, 66236, 26631, 19866, 41929, 20580, 88210]
-
-%% Cell type:markdown id: tags:
-
-## SAX
-
-%% Cell type:code id: tags:
-
-``` python
-# from tslearn.piecewise import SymbolicAggregateApproximation
-
-# t0 = time()
-# sax = SymbolicAggregateApproximation(n_segments=T, alphabet_size_avg=10)
-# sax_data = sax.fit_transform(data)
-# print('Done! Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))
-# sax_preprocess_time = time() - t0
-```
-
-%% Cell type:code id: tags:
-
-``` python
-# t0 = time()
-# all_sax_candidates = []
-# for i, target in enumerate(targets):
-#     t1 = time()
-#     query = sax_data[target]
-#     sax_distances = [np.linalg.norm(query - window) for window in sax_data]
-#     print('Target #{} done! Took {:.2f} seconds ({:.1f} minutes).'.format(i, time() - t1, (time() - t1) / 60))
-#     sax_candidates = sorted(range(len(sax_distances)), key=lambda k: sax_distances[k])
-#     all_sax_candidates.append(sax_candidates)
-# sax_time = time() - t0
-```
-
-%% Cell type:markdown id: tags:
-
-## PSEUDo
-
-%% Cell type:markdown id: tags:
-
-For the LSH algorithm some preprocessing is done to find the right LSH parameters.
-
-%% Cell type:code id: tags:
-
-``` python
-import sys
-
-sys.path.insert(0, '../Flaskserver')
-import importlib
-from pseudo import preprocess
-import _lsh
-
-topk_dtw = []
-
-print('Preprocessing:')
-t0 = time()
-r,a,sd = preprocess(data, data.shape[2])
-print('Preprocessing done. Took {:.2f} seconds ({:.1f} minutes).'.format(time() - t0, (time() - t0) / 60))
-pseudo_preprocess_time = time() - t0
-```
-
-%%%% Output: stream
-
-    Preprocessing:
-    r = 100
-    r = 50.0
-    r = 25.0
-    r = 37.5
-    r = 56.25
-    r = 28.125
-    r = 42.1875
-    r = 21.09375
-    r = 31.640625
-    r = 47.4609375
-    r = 23.73046875
-    r = 35.595703125
-    r = 53.3935546875
-    r = 26.69677734375
-    r = 40.045166015625
-    r = 60.0677490234375
-    r = 30.03387451171875
-    r = 45.050811767578125
-    r = 22.525405883789062
-    r = 33.788108825683594
-    r = 50.68216323852539
-    r = 25.341081619262695
-    r = 38.01162242889404
-    r = 57.017433643341064
-    r = 28.508716821670532
-    r = 42.7630752325058
-    r = 21.3815376162529
-    r = 32.07230642437935
-    r = 48.10845963656902
-    r = 24.05422981828451
-    r = 36.08134472742677
-    r = 54.12201709114015
-    r = 27.061008545570076
-    r = 40.59151281835511
-    r = 60.88726922753267
-    r = 30.443634613766335
-    r = 45.6654519206495
-    r = 22.83272596032475
-    r = 34.24908894048713
-    r = 51.37363341073069
-    r = 25.686816705365345
-    r = 38.53022505804802
-    r = 57.79533758707203
-    r = 28.897668793536013
-    r = 43.34650319030402
-    r = 21.67325159515201
-    r = 32.509877392728015
-    r = 48.76481608909202
-    r = 24.38240804454601
-    r = 36.57361206681902
-    r = 54.86041810022853
-    r = 27.430209050114264
-    r = 41.1453135751714
-    r = 20.5726567875857
-    r = 30.85898518137855
-    r = 46.28847777206782
-    r = 23.14423888603391
-    r = 34.71635832905086
-    r = 52.074537493576294
-    r = 26.037268746788147
-    r = 39.05590312018222
-    r = 58.58385468027333
-    r = 29.291927340136667
-    r = 43.937891010205
-    r = 21.9689455051025
-    r = 32.95341825765375
-    r = 49.43012738648063
-    r = 24.715063693240314
-    r = 37.07259553986047
-    r = 55.60889330979071
-    r = 27.804446654895354
-    r = 41.70666998234303
-    r = 20.853334991171515
-    r = 31.280002486757272
-    r = 46.92000373013591
-    r = 23.460001865067955
-    r = 35.190002797601935
-    r = 52.7850041964029
-    r = 26.39250209820145
-    r = 39.58875314730218
-    r = 59.383129720953264
-    r = 29.691564860476632
-    r = 44.53734729071495
-    r = 22.268673645357474
-    r = 33.403010468036214
-    r = 50.10451570205432
-    r = 25.05225785102716
-    r = 37.57838677654074
-    r = 56.36758016481111
-    r = 28.183790082405555
-    r = 42.27568512360833
-    r = 21.137842561804167
-    r = 31.706763842706252
-    r = 47.56014576405938
-    r = 23.78007288202969
-    r = 35.670109323044535
-    r = 53.505163984566806
-    r = 26.752581992283403
-    r = 40.1288729884251
-    r = 60.19330948263765
-    r = 30.096654741318826
-    r = 45.14498211197824
-    r = 22.57249105598912
-    r = 33.85873658398368
-    r = 50.78810487597552
-    r = 25.39405243798776
-    r = 38.09107865698164
-    r = 57.136617985472455
-    r = 28.568308992736227
-    r = 42.85246348910434
-    r = 21.42623174455217
-    r = 32.13934761682825
-    r = 48.20902142524238
-    r = 24.10451071262119
-    r = 36.15676606893179
-    r = 54.23514910339768
-    r = 27.11757455169884
-    r = 40.67636182754826
-    r = 61.014542741322394
-    r = 30.507271370661197
-    r = 45.760907055991794
-    r = 22.880453527995897
-    r = 34.320680291993845
-    r = 51.481020437990765
-    r = 25.740510218995382
-    r = 38.61076532849307
-    r = 57.91614799273961
-    r = 28.958073996369805
-    r = 43.43711099455471
-    r = 21.718555497277354
-    r = 32.57783324591603
-    r = 48.86674986887405
-    r = 24.433374934437026
-    r = 36.65006240165554
-    r = 54.97509360248331