Fast implementation for debug/testing

Former-commit-id: b9056adb

AngularApp/prototype/src/app/api.service.ts

@@ -6,7 +6,7 @@ export interface RawData {
 }
 
 export interface LshData {
-  candidates: number[][][];
+  candidates: {[bucket: string]: number[]}[];
   average_candidates: number[];
   average_distances: number[];
   distances: number[][][];

AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts

@@ -15,7 +15,7 @@ export class LabelingWindowComponent implements OnInit {
   constructor(private state: StateService) { }
 
   ngOnInit(): void {
-    this.state.onNewTable.subscribe(() => this.showSamples());
+    this.state.onNewLshData.subscribe(() => this.showSamples());
   }
 
   async train() {

AngularApp/prototype/src/app/overview-window/overview-window.component.ts

@@ -26,7 +26,7 @@ export class OverviewWindowComponent implements OnInit {
 
   async ngOnInit(): Promise<void> {
     this.state.onNewData.subscribe(() => this.initializePlot());
-    this.state.onNewTable.subscribe(() => this.updatePlot());
+    // this.state.onNewTable.subscribe(() => this.updatePlot());
   }
 
   async initializePlot() {
@@ -148,7 +148,7 @@ export class OverviewWindowComponent implements OnInit {
     };
     console.log(this.config);
     console.log("showing plot");
-    await this.debugClicked();
+    // await this.debugClicked();
   }
 
   async updatePlot() {
@@ -178,15 +178,15 @@ export class OverviewWindowComponent implements OnInit {
         });
       }
     }
-    // for (const index of this.state.lshData.average_candidates) {
-    //   this.candidateLabels.push([Number(index), 1]);
-    //   this.markers.push({
-    //     type: 'area',
-    //     // BUG: For some reason the range values are multiplied by 10
-    //     range: [Number(index) / 10, (Number(index) + this.state.windowSize) / 10],
-    //     backgroundColor: '#b1a343',
-    //   });
-    // }
+    for (const index of this.state.lshData.average_candidates.slice(0, 100)) {
+      this.candidateLabels.push([Number(index), 0]);
+      this.markers.push({
+        type: 'area',
+        // BUG: For some reason the range values are multiplied by 10
+        range: [Number(index) / 10, (Number(index) + this.state.windowSize) / 10],
+        backgroundColor: '#b1a343',
+      });
+    }
     console.log(this.markers);
     for (const channel of this.config.graphset) {
       if (channel.type === 'line') {

AngularApp/prototype/src/app/progress-view/progress-view.component.css

@@ -27,9 +27,12 @@
   justify-content: center;
 }
 
-mat-slider {
+.slider {
   display: flex;
   justify-content: center;
+}
+
+mat-slider {
   width: 400px;
 }
 

AngularApp/prototype/src/app/progress-view/progress-view.component.ts

@@ -36,7 +36,11 @@ export class ProgressViewComponent implements OnInit {
         marker: {
           color: Object.keys(table).map((key) => {
             return this.getColor(Number(key) / Number(Object.keys(table)[Object.keys(table).length - 1]));
-          })
+          }),
+          line: {
+            color: 'black',
+            width: 0,
+          }
         }
       }],
       layout: {
@@ -142,7 +146,7 @@ export class ProgressViewComponent implements OnInit {
         zeroline: false,
         showticklabels: false,
       },
-      height: 400,
+      height: 350,
       width: 400,
     };
     this.state.queryWindow.forEach((channel: number[], index: number) => {
@@ -157,6 +161,11 @@ export class ProgressViewComponent implements OnInit {
     this._sliderValue = v.value;
     d3.selectAll('circle').transition().style('stroke', undefined);
     d3.select('#node-' + v.value).transition().style('stroke', 'black').style('stroke-width', 20);
+    const data = this.hist;
+    data.data[0].marker.line.width = Object.keys(this.state._averageTable).map((key) => {
+      return Number(key) === v.value ? 4 : 0;
+    });
+    this.hist = data;
   }
 
   public get sliderValue(): number {

AngularApp/prototype/src/app/state.service.ts

@@ -32,6 +32,7 @@ export class StateService {
   public onNewQuery: EventEmitter<void> = new EventEmitter<void>();
   public onNewTable: EventEmitter<void> = new EventEmitter<void>();
   public onNewTableInfo: EventEmitter<void> = new EventEmitter<void>();
+  public onNewLshData: EventEmitter<void> = new EventEmitter<void>();
 
   public onNewLabels: EventEmitter<void> = new EventEmitter<void>();
   public onNewTab: EventEmitter<void> = new EventEmitter<void>();
@@ -66,6 +67,7 @@ export class StateService {
 
   async lshInitial(): Promise<void> {
     this.lshData = await this.api.lshInitial(this._queryWindow);
+    console.log('data loaded');
     this._lshParameters = this.lshData.parameters;
     this.createTable();
   }
@@ -83,6 +85,7 @@ export class StateService {
 
   async getQueryWindow(windowIndex: number | {[index: number]: boolean}): Promise<number[][]> {
     this.queryWindow = await this.api.getQueryWindow(windowIndex);
+    console.log(this.queryWindow);
     return this._queryWindow;
   }
 
@@ -91,25 +94,9 @@ export class StateService {
   }
 
   public createTable() {
-    const table = [];
-    this.lshData.candidates.forEach((groupHash: number[][], i: number) => {
-      groupHash.forEach((candidates: number[], j: number) => {
-        const subTable = {};
-        const length = this.lshData.distances[i][j].length;
-        const median = this.lshData.distances[i][j][Math.ceil(length / 2)];
-        const stepsize = median / 10;
-        const indices: number[] = this.lshData.distances[i][j].map((x) => x > median * 2 ? 19 : Math.floor(x / stepsize));
-        candidates.forEach((candidate: number, index: number) => {
-          if (subTable[indices[index]] === undefined)
-          {
-            subTable[indices[index]] = [];
-          }
-          subTable[indices[index]].push(candidate);
-        });
-        table.push(subTable);
-      });
-    });
-    this.table = table;
+    console.log('setting table param');
+    this.table = this.lshData.candidates;
+    console.log('table param set');
     const averageTable = {};
     const length = this.lshData.average_distances.length;
     const median = this.lshData.average_distances[Math.ceil(length / 2)];
@@ -123,6 +110,7 @@ export class StateService {
       averageTable[indices[index]].push(candidate);
     });
     this._averageTable = averageTable;
+    console.log('table created');
     this.getTableInfo();
   }
 
@@ -139,6 +127,7 @@ export class StateService {
   public set lshData(v: LshData) {
     console.log(v);
     this._lshData = v;
+    this.onNewLshData.emit();
   }
 
   public get lshData(): LshData {
@@ -157,6 +146,7 @@ export class StateService {
   public set table(v: {[bucket: string]: number[]}[]) {
     console.log(v);
     this._table = v;
+    console.log('emitting onNewTable');
     this.onNewTable.emit();
   }
 

AngularApp/prototype/src/app/table-overview/table-overview.component.ts

@@ -73,10 +73,13 @@ export class TableOverviewComponent implements OnInit {
   }
 
   async createHistograms() {
+    console.log('creating table histograms');
     this.subplots = [];
     this.averages = [];
     const tables = this.state.table;
-    tables.forEach((table) => {
+    console.log('start of table histograms');
+    tables.forEach((table, index) => {
+      console.log(index);
       this.subplots.push(
         {
           data: [{
@@ -112,6 +115,7 @@ export class TableOverviewComponent implements OnInit {
         }
       );
     });
+    console.log('tables histogram created');
   }
 
   // async setQuery(data) {

Flaskserver/.idea/workspace.xml

@@ -21,24 +21,14 @@
   <component name="ChangeListManager">
     <list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/api.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/api.service.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.css" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labels/labels.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labels/labels.component.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/main/main.component.html" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.css" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.css" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/progress-view/progress-view.component.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/state.service.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/lsh.o" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/data.pkl" beforeDir="false" />
       <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/processed-data.npy" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/../experiments/.ipynb_checkpoints/MTS test-checkpoint.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/.ipynb_checkpoints/MTS test-checkpoint.ipynb" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../experiments/MTS test.ipynb" beforeDir="false" afterPath="$PROJECT_DIR$/../experiments/MTS test.ipynb" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />

Flaskserver/main.py

@@ -110,6 +110,10 @@ def create_mts_windows():
             windows.append(preprocessing.minmax_scale(data[i], (-1, 1), axis=1))
         print("Preprocessed: " + str(time()-t0))
         np.save('processed-data', windows)
+    # data = np.load('processed-data.npy')
+    # data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
+    # r, a, sd = preprocess(data, 11.5)
+    # np.save('parameters', np.array([r, a, sd]))
     print("Sending response: " + str(time()-t0))
     return '1'
 
@@ -174,8 +178,11 @@ def initialize():
     data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
     query = raw_data["query"]
     query = np.reshape(query, (len(query[0]), len(query)))
+    parameters = np.load('parameters.npy')
+    r = parameters[0]
+    a = parameters[1]
+    sd = parameters[2]
 
-    r, a, sd = preprocess(data)
     candidates, distances, hf = _lsh.lsh(data, query, r, a, sd)
 
     dict = defaultdict(int)
@@ -187,9 +194,23 @@ def initialize():
     average_candidates = list(sorted_dict.keys())
     average_distances = list(sorted_dict.values())
 
+    tables = []
+    candidates = candidates.tolist()
+    for l in range(len(candidates)):
+        for k in range(len(candidates[0])):
+            dict = defaultdict(list)
+            length = len(distances[l][k])
+            median = distances[l][k][math.ceil(length/2)]
+            stepsize = median / 10
+            indices = list(map(lambda x: 19 if x > median * 2 else math.floor(x / stepsize), distances[l][k]))
+            for i in range(len(candidates[0][0])):
+                dict[str(indices[i])].append(candidates[l][k][i])
+            tables.append(dict)
+
+
     response = {
         "hash_functions": hf.tolist(),
-        "candidates": candidates.tolist(),
+        "candidates": tables,
         "average_candidates": np.array(average_candidates).tolist(),
         "average_distances": np.array(average_distances).tolist(),
         "distances": distances.tolist(),
@@ -222,10 +243,34 @@ def update():
     parameters = raw_data["parameters"]
 
     candidates, distances, hf = _lsh.lsh(data, query, parameters[0], parameters[1], parameters[2])
+    dict = defaultdict(int)
+    for l in range(len(candidates)):
+        for k in range(len(candidates[0])):
+            for i in range(len(candidates[0][0])):
+                dict[candidates[l][k][i]] += distances[l][k][i]
+    sorted_dict = {k: v for k, v in sorted(dict.items(), key=lambda item: item[1])}
+    average_candidates = list(sorted_dict.keys())
+    average_distances = list(sorted_dict.values())
+
+    tables = []
+    candidates = candidates.tolist()
+    for l in range(len(candidates)):
+        for k in range(len(candidates[0])):
+            dict = defaultdict(list)
+            length = len(distances[l][k])
+            median = distances[l][k][math.ceil(length/2)]
+            stepsize = median / 10
+            indices = list(map(lambda x: 19 if x > median * 2 else math.floor(x / stepsize), distances[l][k]))
+            for i in range(len(candidates[0][0])):
+                dict[str(indices[i])].append(candidates[l][k][i])
+            tables.append(dict)
+
     response = {
         "hash_functions": hf.tolist(),
+        "candidates": tables,
+        "average_candidates": np.array(average_candidates).tolist(),
+        "average_distances": np.array(average_distances).tolist(),
         "distances": distances.tolist(),
-        "candidates": candidates.tolist()
     }
     response = orjson.dumps(response)
     print('LSH done: ' + str(time()-t0))
@@ -282,7 +327,7 @@ def table_info():
     print("Averages calculated: " + str(time() - t0))
     return response
 
-def preprocess(data, r=1000):
+def preprocess(data, r=10.0):
     # return 0.10882589134534404, 3.1202154563478928, 0.9705780396843037
     # data = np.load('processed-data.npy')
     # data = np.reshape(data, (59999, 20, 120))
@@ -293,6 +338,7 @@ def preprocess(data, r=1000):
     i = 0
     while i < len(data):
         if i % 999 == 0:
+            print(r)
             print(str(i) + ':' + str(len(subset)))
 
         state = 1
@@ -314,7 +360,7 @@ def preprocess(data, r=1000):
             i = 0
 
     # subset = sample(list(range(len(data))), 200)
-
+    print("r = " + str(r))
     dtw_distances = []
     eq_distances = []
     for i, index_1 in enumerate(subset):
@@ -363,7 +409,7 @@ def debug_test_lsh():
     print(data.shape)
     data = np.reshape(data, (len(data), len(data[0][0]), len(data[0])))
 
-    r, a, sd = preprocess(data, 10)
+    r, a, sd = preprocess(data, 11.25)
     create_windows()
     query_n = 1234
     t0 = time()
@@ -388,12 +434,16 @@ def debug_test_lsh():
     print("Calculated exact dtw in: " + str(time()-t0))
     print(topk_dtw[0:20])
 
+    t0 = time()
+    l2distances = [np.linalg.norm(window - query) for window in data]
+    print("Calculated exact l2 in: " + str(time()-t0))
+
     # # distances_ed = [distance.euclidean(query, window) for window in data]
     # # topk_ed = sorted(range(len(distances_ed)), key=lambda k: distances_ed[k])
     #
     accuracy = 0
     for index in topk_dtw[0:20]:
-        if index in candidates[0:200]:
+        if index in candidates:
             accuracy += 1
     print(accuracy)