Commit 97a23c50 authored by Kruyff,D.L.W. (Dylan)'s avatar Kruyff,D.L.W. (Dylan)
Browse files

Query as MTS data (training not working)

parent 369c013d
......@@ -41,6 +41,20 @@ export class ApiService {
return await response.json();
}
// Split data into windows and normalize
async createMtsWindows(parameters): Promise<any> {
const postData = {parameters};
const response = await fetch('http://127.0.0.1:5000/create-mts-windows', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify(postData)
});
return await response.json();
}
// Generate LSH-tables by hashing each window
async createTables(parameters): Promise<any> {
console.log("creating tables");
......@@ -56,6 +70,21 @@ export class ApiService {
return await response.json();
}
// Generate LSH-tables by hashing each window
async createMtsTables(parameters): Promise<any> {
console.log("creating tables");
const postData = {parameters};
const response = await fetch('http://127.0.0.1:5000/create-mts-tables', {
method: 'POST',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: new Blob( [ JSON.stringify(postData) ], { type: 'text/plain' } )
});
return await response.json();
}
async getQueryWindow(window) {
const response = await fetch('http://127.0.0.1:5000/query', {
method: 'POST',
......
......@@ -64,12 +64,12 @@ export class CacheService {
}
async createWindows(): Promise<void> {
await this.api.createWindows(this.parameters);
await this.api.createMtsWindows(this.parameters);
this.onNewWindows.emit();
}
async createTables(): Promise<void> {
this.tables = await this.api.createTables(this.parameters);
this.tables = await this.api.createMtsTables(this.parameters);
}
async getSimilarWindows(): Promise<any> {
......
......@@ -164,7 +164,7 @@ export class OverviewWindowComponent implements OnInit {
}));
console.log(xyInformation);
const index = Math.floor(xyInformation[8].nodeidx / this.service.stepSize);
await this.service.getQueryWindow(this.service.rawValues[0].slice(index, index + this.service.windowSize));
await this.service.getQueryWindow(this.service.rawValues.map((channel) => channel.slice(index, index + this.service.windowSize)));
const temp = {};
temp[index] = true;
this.service.labels = temp;
......
......@@ -21,14 +21,29 @@ export class QueryWindowComponent implements OnInit {
}
initializePlot(): void {
this.plot =
const data = [];
this.service.queryWindow.forEach((channel, index) => {
data.push({
x: [...Array(channel.length).keys()],
y: channel,
type: 'line',
xaxis: 'x',
yaxis: `y${index + 2}`,
});
});
const subplots = [];
this.service.queryWindow.forEach((channel, index) => {
subplots.push([`xy${index + 2}`]);
});
const plot =
{
data: [{
x: [...Array(this.service.queryWindow.length).keys()],
y: this.service.queryWindow,
type: 'line'
}],
data: data,
layout: {
grid: {
rows: this.service.queryWindow.length,
columns: 1,
subplots: subplots,
},
hovermode: 'closest',
autosize: true,
margin: {
......@@ -43,17 +58,22 @@ export class QueryWindowComponent implements OnInit {
titlefont: {
size: 9
},
showlegend: false,
xaxis: {
showgrid: false,
zeroline: false,
showticklabels: false,
},
yaxis: {
zeroline: false,
showticklabels: false,
}
}
};
this.service.queryWindow.forEach((channel, index) => {
plot.layout[`yaxis${index + 2}`] = {
zeroline: false,
showticklabels: false,
};
});
this.plot = plot;
console.log(this.plot);
}
get isQuerySet(): boolean {
......
......@@ -2,11 +2,8 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="556080ba-825c-4b55-a92a-867a4df4fb32" name="Default Changelist" comment="">
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html" afterDir="false" />
<change beforePath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" beforeDir="false" afterPath="$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/data.pkl" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/processed-data.npy" beforeDir="false" afterPath="$PROJECT_DIR$/processed-data.npy" afterDir="false" />
</list>
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
......@@ -19,8 +16,8 @@
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="431">
<caret line="68" column="5" lean-forward="true" selection-start-line="68" selection-start-column="5" selection-end-line="68" selection-end-column="5" />
<state relative-caret-position="199">
<caret line="199" column="31" lean-forward="true" selection-start-line="199" selection-start-column="31" selection-end-line="199" selection-end-column="31" />
<folding>
<element signature="e#0#41#0" expanded="true" />
</folding>
......@@ -48,6 +45,7 @@
<find>dumps</find>
<find>dask</find>
<find>dd</find>
<find>for</find>
</findStrings>
</component>
<component name="Git.Settings">
......@@ -145,15 +143,15 @@
<workItem from="1599487808817" duration="1192000" />
<workItem from="1599578907139" duration="8436000" />
<workItem from="1599661275060" duration="1249000" />
<workItem from="1600001984238" duration="10447000" />
<workItem from="1600001984238" duration="16778000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="104503000" />
<option name="totallyTimeSpent" value="110834000" />
</component>
<component name="ToolWindowManager">
<frame x="-7" y="-7" width="1295" height="695" extended-state="7" />
<frame x="-7" y="-7" width="1295" height="695" extended-state="6" />
<editor active="true" />
<layout>
<window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.28698465" />
......@@ -212,8 +210,8 @@
</entry>
<entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="431">
<caret line="68" column="5" lean-forward="true" selection-start-line="68" selection-start-column="5" selection-end-line="68" selection-end-column="5" />
<state relative-caret-position="199">
<caret line="199" column="31" lean-forward="true" selection-start-line="199" selection-start-column="31" selection-end-line="199" selection-end-column="31" />
<folding>
<element signature="e#0#41#0" expanded="true" />
</folding>
......
......@@ -95,15 +95,23 @@ def create_mts_windows():
if (not os.path.isfile('processed-data.npy')):
filename = 'data.pkl'
df = pd.read_pickle(filename)
values = df.loc[:, 't'].values.astype(str).tolist()
channels = list()
channels.append(df.loc[:, 't'].values.tolist())
channels.append(df.loc[:, 'hu'].values.tolist())
channels.append(df.loc[:, 'td'].values.tolist())
print("Data read: " + str(time()-t0))
raw_data = request.json
window_size = int(raw_data['parameters']["windowsize"])
print("Processing: " + str(time()-t0))
data = [values[i:i+window_size] for i in range(len(values) - window_size)]
data = preprocessing.minmax_scale(data, (-1, 1), axis=1)
data = [([values[i:i+window_size] for values in channels]) for i in range(len(channels[0]) - window_size)]
print("Raw windows: " + str(time()-t0))
windows = []
for i in range(len(data)):
if i % 5000 == 0:
print(i)
windows.append(preprocessing.minmax_scale(data[i], (-1, 1), axis=1))
print("Preprocessed: " + str(time()-t0))
np.save('processed-data', data)
np.save('processed-data', windows)
print("Sending response: " + str(time()-t0))
return '1'
......@@ -144,12 +152,53 @@ def create_tables():
response = orjson.dumps(response)
return response
@app.route('/create-mts-tables', methods=['POST'])
def create_mts_tables():
t0 = time()
print("loading")
data = np.load('processed-data.npy')
print(time()-t0)
raw_data = orjson.loads(request.data)
print(time()-t0)
window_size = int(raw_data['parameters']["windowsize"])
hash_size = int(raw_data['parameters']["hashsize"])
table_size = int(raw_data['parameters']["tablesize"])
data = np.array(data)
print(data.shape)
print('Starting: ' + str(time()-t0))
tables_hash_function = [np.random.uniform(-1, 1, size=(window_size, hash_size)) for _ in range(table_size)]
print('Init time: ' + str(time() - t0))
tables = []
for index in range(table_size):
t1 = time()
table = defaultdict(list)
# signatures_bool = []
# for window in data:
# signatures_bool.append(np.dot([1, 1, 1], np.dot(window, tables_hash_function[index])) > 0)
signatures_bool = np.dot([1, 1, 1], np.dot(data, tables_hash_function[index])) > 0
signatures = [''.join(['1' if x else '0' for x in lst]) for lst in signatures_bool]
for i in range(len(signatures)):
table[signatures[i]].append(i)
print(time()-t1)
tables.append(table)
print('Creation time: ' + str(time() - t0))
hash_functions = np.array(tables_hash_function).tolist()
response = {}
for table_index in range(table_size):
response[str(table_index)] = {
"hash": hash_functions[table_index],
"entries": tables[table_index]
}
response = orjson.dumps(response)
return response
@app.route('/query', methods=['POST'])
def query():
t0 = time()
raw_data = orjson.loads(request.data)
window = raw_data['window']
output = preprocessing.minmax_scale(window, (-1, 1))
output = preprocessing.minmax_scale(window, (-1, 1), axis=1)
response = orjson.dumps(output.tolist())
print("Query done: " + str(time()-t0))
return response
......@@ -165,7 +214,7 @@ def similarity():
output = defaultdict(list)
for t in tables.values():
signature_bool = np.dot(window, t["hash"]) > 0
signature_bool = np.dot([1, 1, 1], np.dot(window, t["hash"])) > 0
signature = ''.join(['1' if x else '0' for x in signature_bool])
neighbours.extend(t["entries"][signature])
neighbours_with_frequency = dict(Counter(neighbours))
......
No preview for this file type
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment