Reintroduced MTS data

Former-commit-id: 6e893f4b

AngularApp/prototype/src/app/api.service.ts

 import { Injectable } from '@angular/core';
 
 export interface RawData {
-  index: string[];
-  values: number[];
+  index: string[][];
+  values: number[][];
 }
 
 export interface LshData {
@@ -29,15 +29,15 @@ export class ApiService {
   constructor() { }
 
   // Read input data
-  async readFile(): Promise<RawData> {
-    const response = await fetch('http://127.0.0.1:5000/read-data');
+  async readFile(): Promise<RawData[]> {
+    const response = await fetch('http://127.0.0.1:5000/read-mts-data');
     return await response.json();
   }
 
   // Split data into windows and normalize
   async createWindows(parameters): Promise<any> {
     const postData = {parameters};
-    const response = await fetch('http://127.0.0.1:5000/create-windows', {
+    const response = await fetch('http://127.0.0.1:5000/create-mts-windows', {
       method: 'POST',
       headers: {
         'Accept': 'application/json',
@@ -74,7 +74,7 @@ export class ApiService {
   }
 
   // Get query window based on windows labeled correct
-  async getQueryWindow(window): Promise<number[]> {
+  async getQueryWindow(window): Promise<number[][]> {
     const response = await fetch('http://127.0.0.1:5000/query', {
       method: 'POST',
       headers: {
@@ -87,7 +87,7 @@ export class ApiService {
   }
 
   // Get data of a window by indices
-  async getWindowByIndices(indices: number[]): Promise<number[][]> {
+  async getWindowByIndices(indices: number[]): Promise<number[][][]> {
     const response = await fetch('http://127.0.0.1:5000/window', {
       method: 'POST',
       headers: {

AngularApp/prototype/src/app/labeling-window/labeling-window.component.ts

@@ -48,45 +48,67 @@ export class LabelingWindowComponent implements OnInit {
       .filter((candidate) => this.state.labels[candidate] !== true)
       .slice(0, this.k);
     this.subplots = [];
-    const values = await this.state.getWindow(this.topk);
-    this.topk.forEach((index, i) => {
-      this.subplots.push(
-        {
-          index,
-          data: [{
-            x: [...Array(values[i].length).keys()],
-            y: values[i],
-            type: 'line'
-          }],
-          layout: {
-            title: `Index: ${index.toString()}`,
-            hovermode: 'closest',
-            autosize: true,
-            margin: {
-              l: 30,
-              r: 30,
-              t: 30,
-              b: 0,
-              pad: 4
-            },
-            height: 150,
-            width: 300,
-            titlefont: {
-              size: 9
-            },
-            xaxis: {
-              showgrid: false,
-              zeroline: false,
-              showticklabels: false,
-            },
-            yaxis: {
-              zeroline: false,
-              showticklabels: false,
-            }
+    const values: number[][][] = await this.state.getWindow(this.topk);
+    for (const idx in this.topk) {
+      const window = values[idx].slice(0, 1);
+      const data = [];
+      window.forEach((channel: number[], index: number) => {
+        data.push({
+          x: [...Array(channel.length).keys()],
+          y: channel,
+          type: 'line',
+          xaxis: 'x',
+          yaxis: `y${index + 2}`,
+        });
+      });
+      const subplots = [];
+      window.forEach((channel: number[], index: number) => {
+        subplots.push([`xy${index + 2}`]);
+      });
+      const plot = {
+        index: this.topk[idx],
+        data: data,
+        layout: {
+          grid: {
+            rows: 1, //this.state.queryWindow.length,
+            columns: 1,
+            subplots: subplots,
+          },
+          showlegend: false,
+          title: `Index: ${this.topk[idx].toString()}`,
+          hovermode: 'closest',
+          autosize: true,
+          margin: {
+            l: 30,
+            r: 30,
+            t: 30,
+            b: 0,
+            pad: 4
+          },
+          height: 100,
+          width: 300,
+          titlefont: {
+            size: 9
+          },
+          xaxis: {
+            showgrid: false,
+            zeroline: false,
+            showticklabels: false,
+          },
+          yaxis: {
+            zeroline: false,
+            showticklabels: false,
           }
         }
-      );
-    });
+      };
+      window.forEach((channel: number[], index: number) => {
+        plot.layout[`yaxis${index + 2}`] = {
+          zeroline: false,
+          showticklabels: false,
+        };
+      });
+      this.subplots.push(plot);
+    }
   }
 
   public get show() {

AngularApp/prototype/src/app/overview-window/overview-window.component.html

-<zingchart-angular [id]="id" [config]="config" (mousewheel)="zoom($event)" (click)="clicked($event)" [height]="150"></zingchart-angular>
+<zingchart-angular #chart [id]="id" [config]="config" (mousewheel)="zoom($event)" (click)="clicked($event)" [height]="150"></zingchart-angular>

AngularApp/prototype/src/app/overview-window/overview-window.component.ts

-import {Component, OnInit} from '@angular/core';
+import {Component, OnInit, ViewChild} from '@angular/core';
 import { StateService } from '../state.service';
 import zingchart from 'zingchart/es6';
 
@@ -8,7 +8,9 @@ import zingchart from 'zingchart/es6';
   styleUrls: ['./overview-window.component.css']
 })
 export class OverviewWindowComponent implements OnInit {
+  @ViewChild('chart') chart;
   public config;
+  public graphset = [];
   public id = 'overview';
   public markers = [];
   public series = [];
@@ -29,61 +31,125 @@ export class OverviewWindowComponent implements OnInit {
   async initializePlot() {
     this.state.queryWindow = undefined;
     // this.debugClicked();
-    this.data = [];
-    for (let i = 0; i < this.state.rawData.values.length; i++) {
-      this.data.push([this.state.rawData.index[i], this.state.rawData.values[i]]);
-    }
-    this.series = [
-      {
-        type: 'line',
-        values: this.data,
-        text: 'Raw Values',
-        zIndex: 5,
-        marker: {
-          alpha: 0.0,
-          zIndex: 10
-        }
-      },
-      {
-        type: 'scatter',
-        values: [],
-        text: 'Good labels',
-        marker: {
-          backgroundColor: '#4caf50'
-        },
-        zIndex: 20,
-      },
-      {
-        type: 'scatter',
-        values: [],
-        text: 'Bad labels',
-        marker: {
-          backgroundColor: '#f44336'
-        },
-        zIndex: 20,
-      }];
-    this.config = {
-      type: 'mixed',
-      preview: {
-        height: '30px',
-        position: '0% 100%',
-        'auto-fit': true
-      },
-      plotarea: {
-        'margin-top': '10px',
-        'margin-bottom': '50%'
-      },
+    this.graphset.push({
+      id: 'preview',
+      type: "scatter",
+      x: 0,
+      y: 0,
       scaleX: {
         zooming: true,
+        minValue: 0,
+        maxValue: this.state.rawData[0].values.length,
         zoomTo: [0, this.state.windowSize],
         'auto-fit': true,
-        markers: this.markers
+        visible: false,
+        guide: {
+          visible: false
+        }
       },
-      'scale-y': {
-        'auto-fit': true
+      height: '30px',
+      scaleY: {
+        maxValue: 1,
+        minValue: -1,
+        visible: false,
+        guide: {
+          visible: false
+        }
+      },
+      preview: {
+        x: 50,
+        y: 10,
+        height: '30px',
       },
-      series: this.series
+      series: [
+        {
+          type: 'scatter',
+          values: [],
+          text: 'Good labels',
+          marker: {
+            backgroundColor: '#4caf50'
+          },
+          zIndex: 3,
+        },
+        {
+          type: 'scatter',
+          values: [],
+          text: 'Bad labels',
+          marker: {
+            backgroundColor: '#f44336'
+          },
+          zIndex: 2,
+        },
+        {
+          type: 'scatter',
+          values: [],
+          text: 'Candidates',
+          marker: {
+            backgroundColor: '#b1a343'
+          },
+          zIndex: 1,
+        }
+      ]
+    });
+    // Initialize channels
+    this.state.rawData.forEach((channel, index) => {
+      if (index !== 0) {
+        return;
+      }
+      const data = [];
+      for (let i = 0; i < channel.values.length; i++) {
+        data.push([i, channel.values[i]]);
+      }
+      this.graphset.push({
+        id: index,
+        x: 0,
+        y: `${75 * index + 50}px`,
+        type: 'line',
+        height: '50px',
+        scaleX: {
+          zooming: true,
+          zoomTo: [0, this.state.windowSize],
+          markers: []
+        },
+        'scale-y': {
+          zooming: false,
+          'auto-fit': true
+        },
+        plotarea: {
+          height: '50px',
+          'margin-top': '0px',
+          'margin-bot': '0px'
+        },
+        series: [{
+          type: 'line',
+          values: data,
+          text: 'Raw Values',
+          zIndex: 5,
+          marker: {
+            alpha: 0.0,
+            zIndex: 10
+          }
+        }]
+      });
+    });
+    zingchart.bind('zingchart-ng-1', 'zoom', (e) => {
+      if (e.graphid !== `zingchart-ng-1-graph-preview`) {
+        return;
+      }
+      for (let i = 1; i < this.graphset.length; i ++) {
+        this.chart.zoomto({
+          graphid: i,
+          xmin: e.xmin,
+          xmax: e.xmax
+        });
+      }
+    });
+    this.config = {
+      layout: `${this.graphset.length}x1`,
+      graphset: this.graphset
     };
+    console.log(this.config);
+    console.log("showing plot");
   }
 
   async updatePlot() {
@@ -170,7 +236,7 @@ export class OverviewWindowComponent implements OnInit {
   }
 
   async debugClicked() {
-    const index = 80503;
+    const index = 1234;
     await this.state.getQueryWindow(index);
     await this.state.lshInitial();
   }

AngularApp/prototype/src/app/query-window/query-window.component.ts

@@ -22,8 +22,8 @@ export class QueryWindowComponent implements OnInit {
 
   initializePlot(): void {
     const data = [{
-        x: [...Array(this.state.queryWindow.length).keys()],
-        y: this.state.queryWindow,
+        x: [...Array(this.state.queryWindow[0].length).keys()],
+        y: this.state.queryWindow[0],
         type: 'line'
       }];
     // if (this.service.distances.length !== 0) {

AngularApp/prototype/src/app/state.service.ts

@@ -7,16 +7,16 @@ import {ApiService, LshData, RawData, TableInfoData} from './api.service';
 export class StateService {
   public loadingProgress: number = 0;
 
-  private _rawData: RawData;
+  private _rawData: RawData[];
   private _lshData: LshData;
   private _tableInfo: TableInfoData;
-  private _queryWindow: number[];
+  private _queryWindow: number[][];
   private _table: {[bucket: string]: number[]};
 
   private _currentTab: number;
   private _labels = {};
   private _sliderValue;
-  private _parameters: number[];
+  private _lshParameters: number[];
 
   public windowSize = 120;
   public nrOfTables = 5;
@@ -63,11 +63,12 @@ export class StateService {
 
   async lshInitial(): Promise<void> {
     this.lshData = await this.api.lshInitial(this._queryWindow);
+    this._lshParameters = this.lshData.parameters;
     this.createTable();
   }
 
   async update(): Promise<void> {
-    this.lshData = await this.api.lshUpdate(this._queryWindow, [], this.lshData.parameters);
+    this.lshData = await this.api.lshUpdate(this._queryWindow, [], this._lshParameters);
     this.createTable();
   }
 
@@ -76,12 +77,12 @@ export class StateService {
     return this.tableInfo;
   }
 
-  async getQueryWindow(windowIndex: number | {[index: number]: boolean}): Promise<number[]> {
+  async getQueryWindow(windowIndex: number | {[index: number]: boolean}): Promise<number[][]> {
     this.queryWindow = await this.api.getQueryWindow(windowIndex);
     return this._queryWindow;
   }
 
-  async getWindow(indices: number[]): Promise<number[][]> {
+  async getWindow(indices: number[]): Promise<number[][][]> {
     return await this.api.getWindowByIndices(indices);
   }
 
@@ -99,12 +100,13 @@ export class StateService {
     this.getTableInfo();
   }
 
-  public set rawData(v: RawData) {
+  public set rawData(v: RawData[]) {
     this._rawData = v;
+    console.log(this._rawData);
     this.onNewData.emit();
   }
 
-  public get rawData(): RawData {
+  public get rawData(): RawData []{
     return this._rawData;
   }
 
@@ -163,15 +165,19 @@ export class StateService {
     return this._sliderValue;
   }
 
-  public set queryWindow(v: number[]) {
+  public set queryWindow(v: number[][]) {
     this._queryWindow = v;
     this.onNewQuery.emit();
   }
 
-  public get queryWindow(): number[] {
+  public get queryWindow(): number[][] {
     return this._queryWindow;
   }
 
+  public get lshParameters(): number[] {
+    return this._lshParameters;
+  }
+
   public get parameters(): {[parameter: string]: number} {
     return {
       windowsize: this.windowSize,

Flaskserver/.idea/workspace.xml

@@ -20,17 +20,11 @@
   </component>
   <component name="ChangeListManager">
     <list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/labeling-window/labeling-window.component.html" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/table-overview/table-overview.component.ts" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/1.csv" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/DailyDelhiClimateTrain.csv" beforeDir="false" />
       <change beforePath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" beforeDir="false" afterPath="$PROJECT_DIR$/build/lib.linux-x86_64-3.8/_lsh.cpython-38-x86_64-linux-gnu.so" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" beforeDir="false" afterPath="$PROJECT_DIR$/build/temp.linux-x86_64-3.8/locality-sensitive-hashing-visual-analytics/lsh-fast/_lsh.o" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/setup.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../environment.yml" beforeDir="false" afterPath="$PROJECT_DIR$/../environment.yml" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -50,7 +44,7 @@
     <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
     <property name="SHARE_PROJECT_CONFIGURATION_FILES" value="true" />
     <property name="WebServerToolWindowFactoryState" value="false" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$/../experiments/utils.py" />
     <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
     <property name="nodejs_npm_path_reset_for_default_project" value="true" />
   </component>
@@ -137,13 +131,21 @@
       <screen x="72" y="27" width="1848" height="1053" />
     </state>
     <state x="686" y="355" width="777" height="403" key="#com.intellij.fileTypes.FileTypeChooser/72.27.1848.1053@72.27.1848.1053" timestamp="1603212805629" />
+    <state x="729" y="302" width="524" height="509" key="#com.intellij.refactoring.safeDelete.UnsafeUsagesDialog" timestamp="1605036046833">
+      <screen x="72" y="27" width="1848" height="1053" />
+    </state>
+    <state x="729" y="302" width="524" height="509" key="#com.intellij.refactoring.safeDelete.UnsafeUsagesDialog/72.27.1848.1053@72.27.1848.1053" timestamp="1605036046833" />
     <state x="479" y="254" width="1200" height="800" key="DiffContextDialog" timestamp="1603129938934">
       <screen x="72" y="27" width="1848" height="1053" />
     </state>
     <state x="479" y="254" width="1200" height="800" key="DiffContextDialog/72.27.1848.1053@72.27.1848.1053" timestamp="1603129938934" />
-    <state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl" timestamp="1603390700256">
+    <state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl" timestamp="1604916700277">
+      <screen x="72" y="27" width="1848" height="1053" />
+    </state>
+    <state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl/72.27.1848.1053@72.27.1848.1053" timestamp="1604916700277" />
+    <state x="659" y="259" width="672" height="678" key="search.everywhere.popup" timestamp="1604929652702">
       <screen x="72" y="27" width="1848" height="1053" />
     </state>
-    <state x="779" y="311" width="424" height="491" key="FileChooserDialogImpl/72.27.1848.1053@72.27.1848.1053" timestamp="1603390700256" />
+    <state x="659" y="259" width="672" height="678" key="search.everywhere.popup/72.27.1848.1053@72.27.1848.1053" timestamp="1604929652702" />
   </component>
 </project>
\ No newline at end of file

Flaskserver/main.py

@@ -2,6 +2,7 @@ from flask import Flask, request
 import numpy as np
 from flask_cors import CORS
 from time import time
+import pandas as pd
 import orjson
 import bigwig
 import bbi
@@ -9,8 +10,12 @@ import _ucrdtw
 import _lsh
 import dtw
 import math
+import dask.dataframe as dd
+import os.path
 from random import sample
-from DBA import performDBA
+from DBA_multivariate import performDBA
+from tslearn.metrics import dtw
+from sklearn import preprocessing
 
 reload = False
 
@@ -36,6 +41,68 @@ def read_data():
     print('Data read: ' + str(time()-t0))
     return response
 
+@app.route('/read-mts-data', methods=['GET'])
+def read_mts_data():
+    filename = 'data.pkl'
+    if (not os.path.isfile(filename)):
+        print("start")
+        df = dd.read_csv("NW_Ground_Stations_2016.csv", usecols=['number_sta', 'date', 't', 'hu', 'td'])
+        print("read file")
+        df = df.loc[df['number_sta'] == 14066001]
+        print("split rows")
+        df = df.compute()
+        df.to_pickle(filename)
+        print("to_pandas")
+    df = pd.read_pickle(filename)
+    df.dropna(subset=['t'], inplace=True)
+    response = [
+        {
+            "index": df.loc[:, 'date'].values.astype(str).tolist(),
+            "values": df.loc[:, 't'].values.tolist()
+        },
+        {
+            "index": df.loc[:, 'date'].values.astype(str).tolist(),
+            "values": df.loc[:, 'hu'].values.tolist()
+        },
+        {
+            "index": df.loc[:, 'date'].values.astype(str).tolist(),
+            "values": df.loc[:, 'td'].values.tolist()
+        }
+    ]
+    print("response ready")
+    response = orjson.dumps(response)
+    return response
+
+@app.route('/create-mts-windows', methods=['POST'])
+def create_mts_windows():
+    t0 = time()
+    if (not os.path.isfile('processed-data.npy')):
+        filename = 'data.pkl'
+        df = pd.read_pickle(filename)
+        channels = list()
+        channels.append(df.loc[:, 't'].fillna(0).values.tolist())