Query as MTS data (training not working)

AngularApp/prototype/src/app/api.service.ts

@@ -41,6 +41,20 @@ export class ApiService {
     return await response.json();
   }
 
+  // Split data into windows and normalize
+  async createMtsWindows(parameters): Promise<any> {
+    const postData = {parameters};
+    const response = await fetch('http://127.0.0.1:5000/create-mts-windows', {
+      method: 'POST',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(postData)
+    });
+    return await response.json();
+  }
+
 // Generate LSH-tables by hashing each window
   async createTables(parameters): Promise<any> {
     console.log("creating tables");
@@ -56,6 +70,21 @@ export class ApiService {
     return await response.json();
   }
 
+  // Generate LSH-tables by hashing each window
+  async createMtsTables(parameters): Promise<any> {
+    console.log("creating tables");
+    const postData = {parameters};
+    const response = await fetch('http://127.0.0.1:5000/create-mts-tables', {
+      method: 'POST',
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json'
+      },
+      body: new Blob( [ JSON.stringify(postData) ], { type: 'text/plain' } )
+    });
+    return await response.json();
+  }
+
   async getQueryWindow(window) {
     const response = await fetch('http://127.0.0.1:5000/query', {
       method: 'POST',

AngularApp/prototype/src/app/cache.service.ts

@@ -64,12 +64,12 @@ export class CacheService {
   }
 
   async createWindows(): Promise<void> {
-    await this.api.createWindows(this.parameters);
+    await this.api.createMtsWindows(this.parameters);
     this.onNewWindows.emit();
   }
 
   async createTables(): Promise<void> {
-    this.tables = await this.api.createTables(this.parameters);
+    this.tables = await this.api.createMtsTables(this.parameters);
   }
 
   async getSimilarWindows(): Promise<any> {

AngularApp/prototype/src/app/overview-window/overview-window.component.ts

@@ -164,7 +164,7 @@ export class OverviewWindowComponent implements OnInit {
     }));
     console.log(xyInformation);
     const index = Math.floor(xyInformation[8].nodeidx / this.service.stepSize);
-    await this.service.getQueryWindow(this.service.rawValues[0].slice(index, index + this.service.windowSize));
+    await this.service.getQueryWindow(this.service.rawValues.map((channel) => channel.slice(index, index + this.service.windowSize)));
     const temp = {};
     temp[index] = true;
     this.service.labels = temp;

AngularApp/prototype/src/app/query-window/query-window.component.ts

@@ -21,14 +21,29 @@ export class QueryWindowComponent implements OnInit {
   }
 
   initializePlot(): void {
-    this.plot =
+    const data = [];
+    this.service.queryWindow.forEach((channel, index) => {
+      data.push({
+        x: [...Array(channel.length).keys()],
+        y: channel,
+        type: 'line',
+        xaxis: 'x',
+        yaxis: `y${index + 2}`,
+      });
+    });
+    const subplots = [];
+    this.service.queryWindow.forEach((channel, index) => {
+      subplots.push([`xy${index + 2}`]);
+    });
+    const plot =
       {
-        data: [{
-          x: [...Array(this.service.queryWindow.length).keys()],
-          y: this.service.queryWindow,
-          type: 'line'
-        }],
+        data: data,
         layout: {
+          grid: {
+            rows: this.service.queryWindow.length,
+            columns: 1,
+            subplots: subplots,
+          },
           hovermode: 'closest',
           autosize: true,
           margin: {
@@ -43,17 +58,22 @@ export class QueryWindowComponent implements OnInit {
           titlefont: {
             size: 9
           },
+          showlegend: false,
           xaxis: {
             showgrid: false,
             zeroline: false,
             showticklabels: false,
-          },
-          yaxis: {
-            zeroline: false,
-            showticklabels: false,
           }
         }
       };
+    this.service.queryWindow.forEach((channel, index) => {
+      plot.layout[`yaxis${index + 2}`] = {
+        zeroline: false,
+        showticklabels: false,
+      };
+    });
+    this.plot = plot;
+    console.log(this.plot);
   }
 
   get isQuerySet(): boolean {

Flaskserver/.idea/workspace.xml

@@ -2,11 +2,8 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/data.pkl" beforeDir="false" />
       <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/processed-data.npy" beforeDir="false" afterPath="$PROJECT_DIR$/processed-data.npy" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
@@ -19,8 +16,8 @@
       <file pinned="false" current-in-tab="true">
         <entry file="file://$PROJECT_DIR$/main.py">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="431">
-              <caret line="68" column="5" lean-forward="true" selection-start-line="68" selection-start-column="5" selection-end-line="68" selection-end-column="5" />
+            <state relative-caret-position="199">
+              <caret line="199" column="31" lean-forward="true" selection-start-line="199" selection-start-column="31" selection-end-line="199" selection-end-column="31" />
               <folding>
                 <element signature="e#0#41#0" expanded="true" />
               </folding>
@@ -48,6 +45,7 @@
       <find>dumps</find>
       <find>dask</find>
       <find>dd</find>
+      <find>for</find>
     </findStrings>
   </component>
   <component name="Git.Settings">
@@ -145,15 +143,15 @@
       <workItem from="1599487808817" duration="1192000" />
       <workItem from="1599578907139" duration="8436000" />
       <workItem from="1599661275060" duration="1249000" />
-      <workItem from="1600001984238" duration="10447000" />
+      <workItem from="1600001984238" duration="16778000" />
     </task>
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="104503000" />
+    <option name="totallyTimeSpent" value="110834000" />
   </component>
   <component name="ToolWindowManager">
-    <frame x="-7" y="-7" width="1295" height="695" extended-state="7" />
+    <frame x="-7" y="-7" width="1295" height="695" extended-state="6" />
     <editor active="true" />
     <layout>
       <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.28698465" />
@@ -212,8 +210,8 @@
     </entry>
     <entry file="file://$PROJECT_DIR$/main.py">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="431">
-          <caret line="68" column="5" lean-forward="true" selection-start-line="68" selection-start-column="5" selection-end-line="68" selection-end-column="5" />
+        <state relative-caret-position="199">
+          <caret line="199" column="31" lean-forward="true" selection-start-line="199" selection-start-column="31" selection-end-line="199" selection-end-column="31" />
           <folding>
             <element signature="e#0#41#0" expanded="true" />
           </folding>

Flaskserver/main.py

@@ -95,15 +95,23 @@ def create_mts_windows():
     if (not os.path.isfile('processed-data.npy')):
         filename = 'data.pkl'
         df = pd.read_pickle(filename)
-        values = df.loc[:, 't'].values.astype(str).tolist()
+        channels = list()
+        channels.append(df.loc[:, 't'].values.tolist())
+        channels.append(df.loc[:, 'hu'].values.tolist())
+        channels.append(df.loc[:, 'td'].values.tolist())
         print("Data read: " + str(time()-t0))
         raw_data = request.json
         window_size = int(raw_data['parameters']["windowsize"])
         print("Processing: " + str(time()-t0))
-        data = [values[i:i+window_size] for i in range(len(values) - window_size)]
-        data = preprocessing.minmax_scale(data, (-1, 1), axis=1)
+        data = [([values[i:i+window_size] for values in channels]) for i in range(len(channels[0]) - window_size)]
+        print("Raw windows: " + str(time()-t0))
+        windows = []
+        for i in range(len(data)):
+            if i % 5000 == 0:
+                print(i)
+            windows.append(preprocessing.minmax_scale(data[i], (-1, 1), axis=1))
         print("Preprocessed: " + str(time()-t0))
-        np.save('processed-data', data)
+        np.save('processed-data', windows)
     print("Sending response: " + str(time()-t0))
     return '1'
 
@@ -144,12 +152,53 @@ def create_tables():
     response = orjson.dumps(response)
     return response
 
+@app.route('/create-mts-tables', methods=['POST'])
+def create_mts_tables():
+    t0 = time()
+    print("loading")
+    data = np.load('processed-data.npy')
+    print(time()-t0)
+    raw_data = orjson.loads(request.data)
+    print(time()-t0)
+    window_size = int(raw_data['parameters']["windowsize"])
+    hash_size = int(raw_data['parameters']["hashsize"])
+    table_size = int(raw_data['parameters']["tablesize"])
+    data = np.array(data)
+    print(data.shape)
+    print('Starting: ' + str(time()-t0))
+    tables_hash_function = [np.random.uniform(-1, 1, size=(window_size, hash_size)) for _ in range(table_size)]
+    print('Init time: ' + str(time() - t0))
+    tables = []
+    for index in range(table_size):
+        t1 = time()
+        table = defaultdict(list)
+        # signatures_bool = []
+        # for window in data:
+        #     signatures_bool.append(np.dot([1, 1, 1], np.dot(window, tables_hash_function[index])) > 0)
+        signatures_bool = np.dot([1, 1, 1], np.dot(data, tables_hash_function[index])) > 0
+        signatures = [''.join(['1' if x else '0' for x in lst]) for lst in signatures_bool]
+        for i in range(len(signatures)):
+            table[signatures[i]].append(i)
+        print(time()-t1)
+        tables.append(table)
+
+    print('Creation time: ' + str(time() - t0))
+    hash_functions = np.array(tables_hash_function).tolist()
+    response = {}
+    for table_index in range(table_size):
+        response[str(table_index)] = {
+            "hash": hash_functions[table_index],
+            "entries": tables[table_index]
+        }
+    response = orjson.dumps(response)
+    return response
+
 @app.route('/query', methods=['POST'])
 def query():
     t0 = time()
     raw_data = orjson.loads(request.data)
     window = raw_data['window']
-    output = preprocessing.minmax_scale(window, (-1, 1))
+    output = preprocessing.minmax_scale(window, (-1, 1), axis=1)
     response = orjson.dumps(output.tolist())
     print("Query done: " + str(time()-t0))
     return response
@@ -165,7 +214,7 @@ def similarity():
     output = defaultdict(list)
 
     for t in tables.values():
-        signature_bool = np.dot(window, t["hash"]) > 0
+        signature_bool = np.dot([1, 1, 1], np.dot(window, t["hash"])) > 0
         signature = ''.join(['1' if x else '0' for x in signature_bool])
         neighbours.extend(t["entries"][signature])
     neighbours_with_frequency = dict(Counter(neighbours))