Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
vig
Sublinear Algorithms for VA
pseudo
Commits
97a23c50
Commit
97a23c50
authored
Sep 16, 2020
by
Kruyff,D.L.W. (Dylan)
Browse files
Query as MTS data (training not working)
parent
369c013d
Changes
8
Hide whitespace changes
Inline
Side-by-side
AngularApp/prototype/src/app/api.service.ts
View file @
97a23c50
...
...
@@ -41,6 +41,20 @@ export class ApiService {
return
await
response
.
json
();
}
// Split data into windows and normalize
async
createMtsWindows
(
parameters
):
Promise
<
any
>
{
const
postData
=
{
parameters
};
const
response
=
await
fetch
(
'
http://127.0.0.1:5000/create-mts-windows
'
,
{
method
:
'
POST
'
,
headers
:
{
'
Accept
'
:
'
application/json
'
,
'
Content-Type
'
:
'
application/json
'
},
body
:
JSON
.
stringify
(
postData
)
});
return
await
response
.
json
();
}
// Generate LSH-tables by hashing each window
async
createTables
(
parameters
):
Promise
<
any
>
{
console
.
log
(
"
creating tables
"
);
...
...
@@ -56,6 +70,21 @@ export class ApiService {
return
await
response
.
json
();
}
// Generate LSH-tables by hashing each window
async
createMtsTables
(
parameters
):
Promise
<
any
>
{
console
.
log
(
"
creating tables
"
);
const
postData
=
{
parameters
};
const
response
=
await
fetch
(
'
http://127.0.0.1:5000/create-mts-tables
'
,
{
method
:
'
POST
'
,
headers
:
{
'
Accept
'
:
'
application/json
'
,
'
Content-Type
'
:
'
application/json
'
},
body
:
new
Blob
(
[
JSON
.
stringify
(
postData
)
],
{
type
:
'
text/plain
'
}
)
});
return
await
response
.
json
();
}
async
getQueryWindow
(
window
)
{
const
response
=
await
fetch
(
'
http://127.0.0.1:5000/query
'
,
{
method
:
'
POST
'
,
...
...
AngularApp/prototype/src/app/cache.service.ts
View file @
97a23c50
...
...
@@ -64,12 +64,12 @@ export class CacheService {
}
async
createWindows
():
Promise
<
void
>
{
await
this
.
api
.
createWindows
(
this
.
parameters
);
await
this
.
api
.
create
Mts
Windows
(
this
.
parameters
);
this
.
onNewWindows
.
emit
();
}
async
createTables
():
Promise
<
void
>
{
this
.
tables
=
await
this
.
api
.
createTables
(
this
.
parameters
);
this
.
tables
=
await
this
.
api
.
create
Mts
Tables
(
this
.
parameters
);
}
async
getSimilarWindows
():
Promise
<
any
>
{
...
...
AngularApp/prototype/src/app/overview-window/overview-window.component.ts
View file @
97a23c50
...
...
@@ -164,7 +164,7 @@ export class OverviewWindowComponent implements OnInit {
}));
console
.
log
(
xyInformation
);
const
index
=
Math
.
floor
(
xyInformation
[
8
].
nodeidx
/
this
.
service
.
stepSize
);
await
this
.
service
.
getQueryWindow
(
this
.
service
.
rawValues
[
0
]
.
slice
(
index
,
index
+
this
.
service
.
windowSize
));
await
this
.
service
.
getQueryWindow
(
this
.
service
.
rawValues
.
map
((
channel
)
=>
channel
.
slice
(
index
,
index
+
this
.
service
.
windowSize
))
)
;
const
temp
=
{};
temp
[
index
]
=
true
;
this
.
service
.
labels
=
temp
;
...
...
AngularApp/prototype/src/app/query-window/query-window.component.ts
View file @
97a23c50
...
...
@@ -21,14 +21,29 @@ export class QueryWindowComponent implements OnInit {
}
initializePlot
():
void
{
this
.
plot
=
const
data
=
[];
this
.
service
.
queryWindow
.
forEach
((
channel
,
index
)
=>
{
data
.
push
({
x
:
[...
Array
(
channel
.
length
).
keys
()],
y
:
channel
,
type
:
'
line
'
,
xaxis
:
'
x
'
,
yaxis
:
`y
${
index
+
2
}
`
,
});
});
const
subplots
=
[];
this
.
service
.
queryWindow
.
forEach
((
channel
,
index
)
=>
{
subplots
.
push
([
`xy
${
index
+
2
}
`
]);
});
const
plot
=
{
data
:
[{
x
:
[...
Array
(
this
.
service
.
queryWindow
.
length
).
keys
()],
y
:
this
.
service
.
queryWindow
,
type
:
'
line
'
}],
data
:
data
,
layout
:
{
grid
:
{
rows
:
this
.
service
.
queryWindow
.
length
,
columns
:
1
,
subplots
:
subplots
,
},
hovermode
:
'
closest
'
,
autosize
:
true
,
margin
:
{
...
...
@@ -43,17 +58,22 @@ export class QueryWindowComponent implements OnInit {
titlefont
:
{
size
:
9
},
showlegend
:
false
,
xaxis
:
{
showgrid
:
false
,
zeroline
:
false
,
showticklabels
:
false
,
},
yaxis
:
{
zeroline
:
false
,
showticklabels
:
false
,
}
}
};
this
.
service
.
queryWindow
.
forEach
((
channel
,
index
)
=>
{
plot
.
layout
[
`yaxis
${
index
+
2
}
`
]
=
{
zeroline
:
false
,
showticklabels
:
false
,
};
});
this
.
plot
=
plot
;
console
.
log
(
this
.
plot
);
}
get
isQuerySet
():
boolean
{
...
...
Flaskserver/.idea/workspace.xml
View file @
97a23c50
...
...
@@ -2,11 +2,8 @@
<project
version=
"4"
>
<component
name=
"ChangeListManager"
>
<list
default=
"true"
id=
"556080ba-825c-4b55-a92a-867a4df4fb32"
name=
"Default Changelist"
comment=
""
>
<change
beforePath=
"$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.html"
afterDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/../AngularApp/prototype/src/app/overview-window/overview-window.component.ts"
afterDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/.idea/workspace.xml"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/.idea/workspace.xml"
afterDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/data.pkl"
beforeDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/main.py"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/main.py"
afterDir=
"false"
/>
<change
beforePath=
"$PROJECT_DIR$/processed-data.npy"
beforeDir=
"false"
afterPath=
"$PROJECT_DIR$/processed-data.npy"
afterDir=
"false"
/>
</list>
<option
name=
"EXCLUDED_CONVERTED_TO_IGNORED"
value=
"true"
/>
<option
name=
"SHOW_DIALOG"
value=
"false"
/>
...
...
@@ -19,8 +16,8 @@
<file
pinned=
"false"
current-in-tab=
"true"
>
<entry
file=
"file://$PROJECT_DIR$/main.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"
431
"
>
<caret
line=
"
68
"
column=
"
5
"
lean-forward=
"true"
selection-start-line=
"
68
"
selection-start-column=
"
5
"
selection-end-line=
"
68
"
selection-end-column=
"
5
"
/>
<state
relative-caret-position=
"
199
"
>
<caret
line=
"
199
"
column=
"
31
"
lean-forward=
"true"
selection-start-line=
"
199
"
selection-start-column=
"
31
"
selection-end-line=
"
199
"
selection-end-column=
"
31
"
/>
<folding>
<element
signature=
"e#0#41#0"
expanded=
"true"
/>
</folding>
...
...
@@ -48,6 +45,7 @@
<find>
dumps
</find>
<find>
dask
</find>
<find>
dd
</find>
<find>
for
</find>
</findStrings>
</component>
<component
name=
"Git.Settings"
>
...
...
@@ -145,15 +143,15 @@
<workItem
from=
"1599487808817"
duration=
"1192000"
/>
<workItem
from=
"1599578907139"
duration=
"8436000"
/>
<workItem
from=
"1599661275060"
duration=
"1249000"
/>
<workItem
from=
"1600001984238"
duration=
"1
0447
000"
/>
<workItem
from=
"1600001984238"
duration=
"1
6778
000"
/>
</task>
<servers
/>
</component>
<component
name=
"TimeTrackingManager"
>
<option
name=
"totallyTimeSpent"
value=
"1
04503
000"
/>
<option
name=
"totallyTimeSpent"
value=
"1
10834
000"
/>
</component>
<component
name=
"ToolWindowManager"
>
<frame
x=
"-7"
y=
"-7"
width=
"1295"
height=
"695"
extended-state=
"
7
"
/>
<frame
x=
"-7"
y=
"-7"
width=
"1295"
height=
"695"
extended-state=
"
6
"
/>
<editor
active=
"true"
/>
<layout>
<window_info
active=
"true"
content_ui=
"combo"
id=
"Project"
order=
"0"
visible=
"true"
weight=
"0.28698465"
/>
...
...
@@ -212,8 +210,8 @@
</entry>
<entry
file=
"file://$PROJECT_DIR$/main.py"
>
<provider
selected=
"true"
editor-type-id=
"text-editor"
>
<state
relative-caret-position=
"
431
"
>
<caret
line=
"
68
"
column=
"
5
"
lean-forward=
"true"
selection-start-line=
"
68
"
selection-start-column=
"
5
"
selection-end-line=
"
68
"
selection-end-column=
"
5
"
/>
<state
relative-caret-position=
"
199
"
>
<caret
line=
"
199
"
column=
"
31
"
lean-forward=
"true"
selection-start-line=
"
199
"
selection-start-column=
"
31
"
selection-end-line=
"
199
"
selection-end-column=
"
31
"
/>
<folding>
<element
signature=
"e#0#41#0"
expanded=
"true"
/>
</folding>
...
...
Flaskserver/__pycache__/main.cpython-38.pyc
View file @
97a23c50
No preview for this file type
Flaskserver/main.py
View file @
97a23c50
...
...
@@ -95,15 +95,23 @@ def create_mts_windows():
if
(
not
os
.
path
.
isfile
(
'processed-data.npy'
)):
filename
=
'data.pkl'
df
=
pd
.
read_pickle
(
filename
)
values
=
df
.
loc
[:,
't'
].
values
.
astype
(
str
).
tolist
()
channels
=
list
()
channels
.
append
(
df
.
loc
[:,
't'
].
values
.
tolist
())
channels
.
append
(
df
.
loc
[:,
'hu'
].
values
.
tolist
())
channels
.
append
(
df
.
loc
[:,
'td'
].
values
.
tolist
())
print
(
"Data read: "
+
str
(
time
()
-
t0
))
raw_data
=
request
.
json
window_size
=
int
(
raw_data
[
'parameters'
][
"windowsize"
])
print
(
"Processing: "
+
str
(
time
()
-
t0
))
data
=
[
values
[
i
:
i
+
window_size
]
for
i
in
range
(
len
(
values
)
-
window_size
)]
data
=
preprocessing
.
minmax_scale
(
data
,
(
-
1
,
1
),
axis
=
1
)
data
=
[([
values
[
i
:
i
+
window_size
]
for
values
in
channels
])
for
i
in
range
(
len
(
channels
[
0
])
-
window_size
)]
print
(
"Raw windows: "
+
str
(
time
()
-
t0
))
windows
=
[]
for
i
in
range
(
len
(
data
)):
if
i
%
5000
==
0
:
print
(
i
)
windows
.
append
(
preprocessing
.
minmax_scale
(
data
[
i
],
(
-
1
,
1
),
axis
=
1
))
print
(
"Preprocessed: "
+
str
(
time
()
-
t0
))
np
.
save
(
'processed-data'
,
data
)
np
.
save
(
'processed-data'
,
windows
)
print
(
"Sending response: "
+
str
(
time
()
-
t0
))
return
'1'
...
...
@@ -144,12 +152,53 @@ def create_tables():
response
=
orjson
.
dumps
(
response
)
return
response
@
app
.
route
(
'/create-mts-tables'
,
methods
=
[
'POST'
])
def
create_mts_tables
():
t0
=
time
()
print
(
"loading"
)
data
=
np
.
load
(
'processed-data.npy'
)
print
(
time
()
-
t0
)
raw_data
=
orjson
.
loads
(
request
.
data
)
print
(
time
()
-
t0
)
window_size
=
int
(
raw_data
[
'parameters'
][
"windowsize"
])
hash_size
=
int
(
raw_data
[
'parameters'
][
"hashsize"
])
table_size
=
int
(
raw_data
[
'parameters'
][
"tablesize"
])
data
=
np
.
array
(
data
)
print
(
data
.
shape
)
print
(
'Starting: '
+
str
(
time
()
-
t0
))
tables_hash_function
=
[
np
.
random
.
uniform
(
-
1
,
1
,
size
=
(
window_size
,
hash_size
))
for
_
in
range
(
table_size
)]
print
(
'Init time: '
+
str
(
time
()
-
t0
))
tables
=
[]
for
index
in
range
(
table_size
):
t1
=
time
()
table
=
defaultdict
(
list
)
# signatures_bool = []
# for window in data:
# signatures_bool.append(np.dot([1, 1, 1], np.dot(window, tables_hash_function[index])) > 0)
signatures_bool
=
np
.
dot
([
1
,
1
,
1
],
np
.
dot
(
data
,
tables_hash_function
[
index
]))
>
0
signatures
=
[
''
.
join
([
'1'
if
x
else
'0'
for
x
in
lst
])
for
lst
in
signatures_bool
]
for
i
in
range
(
len
(
signatures
)):
table
[
signatures
[
i
]].
append
(
i
)
print
(
time
()
-
t1
)
tables
.
append
(
table
)
print
(
'Creation time: '
+
str
(
time
()
-
t0
))
hash_functions
=
np
.
array
(
tables_hash_function
).
tolist
()
response
=
{}
for
table_index
in
range
(
table_size
):
response
[
str
(
table_index
)]
=
{
"hash"
:
hash_functions
[
table_index
],
"entries"
:
tables
[
table_index
]
}
response
=
orjson
.
dumps
(
response
)
return
response
@
app
.
route
(
'/query'
,
methods
=
[
'POST'
])
def
query
():
t0
=
time
()
raw_data
=
orjson
.
loads
(
request
.
data
)
window
=
raw_data
[
'window'
]
output
=
preprocessing
.
minmax_scale
(
window
,
(
-
1
,
1
))
output
=
preprocessing
.
minmax_scale
(
window
,
(
-
1
,
1
)
,
axis
=
1
)
response
=
orjson
.
dumps
(
output
.
tolist
())
print
(
"Query done: "
+
str
(
time
()
-
t0
))
return
response
...
...
@@ -165,7 +214,7 @@ def similarity():
output
=
defaultdict
(
list
)
for
t
in
tables
.
values
():
signature_bool
=
np
.
dot
(
window
,
t
[
"hash"
])
>
0
signature_bool
=
np
.
dot
([
1
,
1
,
1
],
np
.
dot
(
window
,
t
[
"hash"
])
)
>
0
signature
=
''
.
join
([
'1'
if
x
else
'0'
for
x
in
signature_bool
])
neighbours
.
extend
(
t
[
"entries"
][
signature
])
neighbours_with_frequency
=
dict
(
Counter
(
neighbours
))
...
...
Flaskserver/processed-data.npy
LFS
View file @
97a23c50
No preview for this file type
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment