Commit 97a23c50 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Query as MTS data (training not working)

parent 369c013d
...@@ -41,6 +41,20 @@ export class ApiService { ...@@ -41,6 +41,20 @@ export class ApiService {
return await response.json(); return await response.json();
} }
// Split data into windows and normalize
async createMtsWindows(parameters): Promise<any> {
const postData = {parameters};
const response = await fetch('http://127.0.0.1:5000/create-mts-windows', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify(postData)
});
return await response.json();
}
// Generate LSH-tables by hashing each window // Generate LSH-tables by hashing each window
async createTables(parameters): Promise<any> { async createTables(parameters): Promise<any> {
console.log("creating tables"); console.log("creating tables");
...@@ -56,6 +70,21 @@ export class ApiService { ...@@ -56,6 +70,21 @@ export class ApiService {
return await response.json(); return await response.json();
} }
// Generate LSH-tables by hashing each window
async createMtsTables(parameters): Promise<any> {
console.log("creating tables");
const postData = {parameters};
const response = await fetch('http://127.0.0.1:5000/create-mts-tables', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: new Blob( [ JSON.stringify(postData) ], { type: 'text/plain' } )
});
return await response.json();
}
async getQueryWindow(window) { async getQueryWindow(window) {
const response = await fetch('http://127.0.0.1:5000/query', { const response = await fetch('http://127.0.0.1:5000/query', {
method: 'POST', method: 'POST',
......
...@@ -64,12 +64,12 @@ export class CacheService { ...@@ -64,12 +64,12 @@ export class CacheService {
} }
async createWindows(): Promise<void> { async createWindows(): Promise<void> {
await this.api.createWindows(this.parameters); await this.api.createMtsWindows(this.parameters);
this.onNewWindows.emit(); this.onNewWindows.emit();
} }
async createTables(): Promise<void> { async createTables(): Promise<void> {
this.tables = await this.api.createTables(this.parameters); this.tables = await this.api.createMtsTables(this.parameters);
} }
async getSimilarWindows(): Promise<any> { async getSimilarWindows(): Promise<any> {
......
...@@ -164,7 +164,7 @@ export class OverviewWindowComponent implements OnInit { ...@@ -164,7 +164,7 @@ export class OverviewWindowComponent implements OnInit {
})); }));
console.log(xyInformation); console.log(xyInformation);
const index = Math.floor(xyInformation[8].nodeidx / this.service.stepSize); const index = Math.floor(xyInformation[8].nodeidx / this.service.stepSize);
await this.service.getQueryWindow(this.service.rawValues[0].slice(index, index + this.service.windowSize)); await this.service.getQueryWindow(this.service.rawValues.map((channel) => channel.slice(index, index + this.service.windowSize)));
const temp = {}; const temp = {};
temp[index] = true; temp[index] = true;
this.service.labels = temp; this.service.labels = temp;
......
...@@ -21,14 +21,29 @@ export class QueryWindowComponent implements OnInit { ...@@ -21,14 +21,29 @@ export class QueryWindowComponent implements OnInit {
} }
initializePlot(): void { initializePlot(): void {
this.plot = const data = [];
this.service.queryWindow.forEach((channel, index) => {
data.push({
x: [...Array(channel.length).keys()],
y: channel,
type: 'line',
xaxis: 'x',
yaxis: `y${index + 2}`,
});
});
const subplots = [];
this.service.queryWindow.forEach((channel, index) => {
subplots.push([`xy${index + 2}`]);
});
const plot =
{ {
data: [{ data: data,
x: [...Array(this.service.queryWindow.length).keys()],
y: this.service.queryWindow,
type: 'line'
}],
layout: { layout: {
grid: {
rows: this.service.queryWindow.length,
columns: 1,
subplots: subplots,
},
hovermode: 'closest', hovermode: 'closest',
autosize: true, autosize: true,
margin: { margin: {
...@@ -43,17 +58,22 @@ export class QueryWindowComponent implements OnInit { ...@@ -43,17 +58,22 @@ export class QueryWindowComponent implements OnInit {
titlefont: { titlefont: {
size: 9 size: 9
}, },
showlegend: false,
xaxis: { xaxis: {
showgrid: false, showgrid: false,
zeroline: false, zeroline: false,
showticklabels: false, showticklabels: false,
},
yaxis: {
zeroline: false,
showticklabels: false,
} }
} }
}; };
this.service.queryWindow.forEach((channel, index) => {
plot.layout[`yaxis${index + 2}`] = {
zeroline: false,
showticklabels: false,
};
});
this.plot = plot;
console.log(this.plot);
} }
get isQuerySet(): boolean { get isQuerySet(): boolean {
......
...@@ -2,11 +2,8 @@ ...@@ -2,11 +2,8 @@
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment=""> <list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/data.pkl" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" /> <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/processed-data.npy" beforeDir="false" afterPath="$PROJECT_DIR$/processed-data.npy" afterDir="false" />
</list> </list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
...@@ -19,8 +16,8 @@ ...@@ -19,8 +16,8 @@
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/main.py"> <entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="431"> <state relative-caret-position="199">
<caret line="68" column="5" lean-forward="true" selection-start-line="68" selection-start-column="5" selection-end-line="68" selection-end-column="5" /> <caret line="199" column="31" lean-forward="true" selection-start-line="199" selection-start-column="31" selection-end-line="199" selection-end-column="31" />
<folding> <folding>
<element signature="e#0#41#0" expanded="true" /> <element signature="e#0#41#0" expanded="true" />
</folding> </folding>
...@@ -48,6 +45,7 @@ ...@@ -48,6 +45,7 @@
<find>dumps</find> <find>dumps</find>
<find>dask</find> <find>dask</find>
<find>dd</find> <find>dd</find>
<find>for</find>
</findStrings> </findStrings>
</component> </component>
<component name="Git.Settings"> <component name="Git.Settings">
...@@ -145,15 +143,15 @@ ...@@ -145,15 +143,15 @@
<workItem from="1599487808817" duration="1192000" /> <workItem from="1599487808817" duration="1192000" />
<workItem from="1599578907139" duration="8436000" /> <workItem from="1599578907139" duration="8436000" />
<workItem from="1599661275060" duration="1249000" /> <workItem from="1599661275060" duration="1249000" />
<workItem from="1600001984238" duration="10447000" /> <workItem from="1600001984238" duration="16778000" />
</task> </task>
<servers /> <servers />
</component> </component>
<component name="TimeTrackingManager"> <component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="104503000" /> <option name="totallyTimeSpent" value="110834000" />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="-7" y="-7" width="1295" height="695" extended-state="7" /> <frame x="-7" y="-7" width="1295" height="695" extended-state="6" />
<editor active="true" /> <editor active="true" />
<layout> <layout>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.28698465" /> <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.28698465" />
...@@ -212,8 +210,8 @@ ...@@ -212,8 +210,8 @@
</entry> </entry>
<entry file="file://$PROJECT_DIR$/main.py"> <entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="431"> <state relative-caret-position="199">
<caret line="68" column="5" lean-forward="true" selection-start-line="68" selection-start-column="5" selection-end-line="68" selection-end-column="5" /> <caret line="199" column="31" lean-forward="true" selection-start-line="199" selection-start-column="31" selection-end-line="199" selection-end-column="31" />
<folding> <folding>
<element signature="e#0#41#0" expanded="true" /> <element signature="e#0#41#0" expanded="true" />
</folding> </folding>
......
...@@ -95,15 +95,23 @@ def create_mts_windows(): ...@@ -95,15 +95,23 @@ def create_mts_windows():
if (not os.path.isfile('processed-data.npy')): if (not os.path.isfile('processed-data.npy')):
filename = 'data.pkl' filename = 'data.pkl'
df = pd.read_pickle(filename) df = pd.read_pickle(filename)
values = df.loc[:, 't'].values.astype(str).tolist() channels = list()
channels.append(df.loc[:, 't'].values.tolist())
channels.append(df.loc[:, 'hu'].values.tolist())
channels.append(df.loc[:, 'td'].values.tolist())
print("Data read: " + str(time()-t0)) print("Data read: " + str(time()-t0))
raw_data = request.json raw_data = request.json
window_size = int(raw_data['parameters']["windowsize"]) window_size = int(raw_data['parameters']["windowsize"])
print("Processing: " + str(time()-t0)) print("Processing: " + str(time()-t0))
data = [values[i:i+window_size] for i in range(len(values) - window_size)] data = [([values[i:i+window_size] for values in channels]) for i in range(len(channels[0]) - window_size)]
data = preprocessing.minmax_scale(data, (-1, 1), axis=1) print("Raw windows: " + str(time()-t0))
windows = []
for i in range(len(data)):
if i % 5000 == 0:
print(i)
windows.append(preprocessing.minmax_scale(data[i], (-1, 1), axis=1))
print("Preprocessed: " + str(time()-t0)) print("Preprocessed: " + str(time()-t0))
np.save('processed-data', data) np.save('processed-data', windows)
print("Sending response: " + str(time()-t0)) print("Sending response: " + str(time()-t0))
return '1' return '1'
...@@ -144,12 +152,53 @@ def create_tables(): ...@@ -144,12 +152,53 @@ def create_tables():
response = orjson.dumps(response) response = orjson.dumps(response)
return response return response
@app.route('/create-mts-tables', methods=['POST'])
def create_mts_tables():
t0 = time()
print("loading")
data = np.load('processed-data.npy')
print(time()-t0)
raw_data = orjson.loads(request.data)
print(time()-t0)
window_size = int(raw_data['parameters']["windowsize"])
hash_size = int(raw_data['parameters']["hashsize"])
table_size = int(raw_data['parameters']["tablesize"])
data = np.array(data)
print(data.shape)
print('Starting: ' + str(time()-t0))
tables_hash_function = [np.random.uniform(-1, 1, size=(window_size, hash_size)) for _ in range(table_size)]
print('Init time: ' + str(time() - t0))
tables = []
for index in range(table_size):
t1 = time()
table = defaultdict(list)
# signatures_bool = []
# for window in data:
# signatures_bool.append(np.dot([1, 1, 1], np.dot(window, tables_hash_function[index])) > 0)
signatures_bool = np.dot([1, 1, 1], np.dot(data, tables_hash_function[index])) > 0
signatures = [''.join(['1' if x else '0' for x in lst]) for lst in signatures_bool]
for i in range(len(signatures)):
table[signatures[i]].append(i)
print(time()-t1)
tables.append(table)
print('Creation time: ' + str(time() - t0))
hash_functions = np.array(tables_hash_function).tolist()
response = {}
for table_index in range(table_size):
response[str(table_index)] = {
"hash": hash_functions[table_index],
"entries": tables[table_index]
}
response = orjson.dumps(response)
return response
@app.route('/query', methods=['POST']) @app.route('/query', methods=['POST'])
def query(): def query():
t0 = time() t0 = time()
raw_data = orjson.loads(request.data) raw_data = orjson.loads(request.data)
window = raw_data['window'] window = raw_data['window']
output = preprocessing.minmax_scale(window, (-1, 1)) output = preprocessing.minmax_scale(window, (-1, 1), axis=1)
response = orjson.dumps(output.tolist()) response = orjson.dumps(output.tolist())
print("Query done: " + str(time()-t0)) print("Query done: " + str(time()-t0))
return response return response
...@@ -165,7 +214,7 @@ def similarity(): ...@@ -165,7 +214,7 @@ def similarity():
output = defaultdict(list) output = defaultdict(list)
for t in tables.values(): for t in tables.values():
signature_bool = np.dot(window, t["hash"]) > 0 signature_bool = np.dot([1, 1, 1], np.dot(window, t["hash"])) > 0
signature = ''.join(['1' if x else '0' for x in signature_bool]) signature = ''.join(['1' if x else '0' for x in signature_bool])
neighbours.extend(t["entries"][signature]) neighbours.extend(t["entries"][signature])
neighbours_with_frequency = dict(Counter(neighbours)) neighbours_with_frequency = dict(Counter(neighbours))
......
No preview for this file type
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment