Skip to content
Snippets Groups Projects
Commit b4ad0142 authored by Lelieveld,J.R.J. (Joris)'s avatar Lelieveld,J.R.J. (Joris)
Browse files

Merge branch 'feature/274-relation-supportv3' into 'develop'

Query relation parser now deletes duplicates, result parser handles new format

See merge request datastrophe/microservices-backbone/query-service!11
parents e193dd6e eae2530c
No related branches found
No related tags found
No related merge requests found
......@@ -29,16 +29,16 @@ type parsedJSON struct {
}
type returnStruct struct {
Entities []int
Relations []int
Entities []int //`json:"entities"`
Relations []int //`json:"relation"`
}
type entityStruct struct {
Type string
Constraints []constraintStruct
Type string //`json:"type"`
Constraints []constraintStruct //`json:"constraints"`
}
type relationStruct struct {
Type string
Type string //`json:"type"`
EntityFrom int
EntityTo int
Depth searchDepthStruct
......@@ -50,10 +50,10 @@ type searchDepthStruct struct {
}
type constraintStruct struct {
Attribute string
Value string
DataType string
MatchType string
Attribute string //`json:"attribute"`
Value string //`json:"value"`
DataType string //`json:"dataType"`
MatchType string //`json:"matchType"`
}
// ConvertQuery converts a json string to an AQL query
......@@ -65,18 +65,6 @@ func (s *Service) ConvertQuery(jsonMsg *[]byte) (*string, error) {
return nil, err
}
//Per node query
// per constraint
//relations koppelen ze samen
//return statement
// var result *string
// if len(jsonStruct.Return.Relations) > 0 {
// result = createEdgeQuery(jsonStruct, "e0")
// } else {
// result = createAllNodesQuery(jsonStruct.Return.Entities, jsonStruct.Entities)
// }
result := createQuery(jsonStruct)
return result, nil
}
......@@ -92,104 +80,182 @@ func convertJSONToStruct(jsonMsg *[]byte) (*parsedJSON, error) {
return &jsonStruct, nil
}
func createQuery(jsQuery *parsedJSON) *string {
func createQuery(jsonQuery *parsedJSON) *string {
// GROTE SIDENOTE:
// Vrij zeker dat een query waar alléén edges worden opgevraagd (#4)
// niet wordt gesupport door zowel de result parser als de frontend reciever
jsonQuery := *jsQuery
// TODO:
// NODES
// EDGES (als ze er zijn)
// RETURN STATEMENT
var ret string
for k, v := range jsonQuery.Entities {
name := fmt.Sprintf("n%v", k)
ret += *createNodeLet(&v, &name)
}
// niet wordt gesupport door zowel de result parser als de frontend receiver
var (
relationsToReturn []string
nodesToReturn []string
nodeUnion string
relationUnion string
)
// Loop over all relations
ret := ""
for i, relation := range jsonQuery.Relations {
relationName := fmt.Sprintf("r%v", i)
var onlyEdge bool
if len(jsonQuery.Entities) == 0 {
onlyEdge = true
if relation.EntityFrom != -1 {
// if there is a from-node
// create the let for this node
fromName := fmt.Sprintf("n%v", relation.EntityFrom)
ret += *createNodeLet(&jsonQuery.Entities[relation.EntityFrom], &fromName)
ret += *createRelationLetWithFromEntity(&relation, relationName, &jsonQuery.Entities)
} else if relation.EntityTo != -1 {
// if there is only a to-node
toName := fmt.Sprintf("n%v", relation.EntityTo)
ret += *createNodeLet(&jsonQuery.Entities[relation.EntityTo], &toName)
ret += *createRelationLetWithOnlyToEntity(&relation, relationName, &jsonQuery.Entities)
// Add this relation to the list
} else {
fmt.Println("Relation-only queries are currently not supported")
continue
}
// Add this relation to the list
relationsToReturn = append(relationsToReturn, relationName)
}
for k, v := range jsonQuery.Relations {
name := fmt.Sprintf("e%v", k)
ret += *createEdgeLet(&v, &name, onlyEdge)
// Add node let statements for nodes that are not yet returned
// Create a set from all the entity-from's and entity-to's, to check if they are returned
nodeSet := make(map[int]bool)
for _, relation := range jsonQuery.Relations {
nodeSet[relation.EntityFrom] = true
nodeSet[relation.EntityTo] = true
}
finalReturn := "RETURN {"
// Check if the entities to return are already returned
for _, entityIndex := range jsonQuery.Return.Entities {
if !nodeSet[entityIndex] {
// If not, return this node
name := fmt.Sprintf("n%v", entityIndex)
ret += *createNodeLet(&jsonQuery.Entities[entityIndex], &name)
// If there is an edge you can only return the edge, because the nodes are included.
// If there are no edges, return the nodes
if l := len(jsonQuery.Relations); l > 0 {
for k := range jsonQuery.Relations {
finalReturn += fmt.Sprintf("e%v:e%v", k, k)
if k < l-1 {
finalReturn += ","
}
}
} else {
for k := range jsonQuery.Entities {
finalReturn += fmt.Sprintf("n%v:n%v", k, k)
if k < l-1 {
finalReturn += ","
}
// Add this node to the list
nodesToReturn = append(nodesToReturn, name)
}
}
finalReturn += "}"
ret += finalReturn
// Create UNION statements that create unique lists of all the nodes and relations
nodeUnion = "\nLET nodes = first(RETURN UNION_DISTINCT("
for _, relation := range relationsToReturn {
nodeUnion += fmt.Sprintf("flatten(%v[**].vertices), ", relation)
}
for _, node := range nodesToReturn {
nodeUnion += fmt.Sprintf("%v,", node)
}
nodeUnion += "[],[]))\n"
relationUnion = "LET edges = first(RETURN UNION_DISTINCT("
for _, relation := range relationsToReturn {
relationUnion += fmt.Sprintf("flatten(%v[**].edges), ", relation)
}
relationUnion += "[],[]))\n"
ret += nodeUnion + relationUnion
ret += "RETURN {\"vertices\":nodes, \"edges\":edges }"
return &ret
}
func createNodeLet(node *entityStruct, name *string) *string {
header := fmt.Sprintf("LET %v = (\n\tFOR x IN %v \n", *name, node.Type)
footer := "\tRETURN x\n)\n"
constraints := *createConstraints(&node.Constraints, false, false)
constraints := *createConstraintStatements(&node.Constraints, "x", false)
ret := header + constraints + footer
return &ret
}
func createEdgeLet(edge *relationStruct, name *string, onlyEdge bool) *string {
var (
header string
forEdge string
forSecondNode string
)
func createRelationLetWithFromEntity(relation *relationStruct, name string, entities *[]entityStruct) *string {
header := fmt.Sprintf("LET %v = (\n\tFOR x IN n%v \n", name, relation.EntityFrom)
forStatement := fmt.Sprintf("\tFOR v, e, p IN %v..%v OUTBOUND x %s \n", relation.Depth.Min, relation.Depth.Max, relation.Type)
footer := "\n\tLIMIT 100\n\tRETURN { vertices: p.vertices[*], edges: p.edges[*] }\n)\n"
// Guarantees that there is no path returned with a duplicate edge
// This way there are no cycle paths possible, TODO: more research about this needed
optionStmtn := "\tOPTIONS { uniqueEdges: \"path\" }\n"
// WICKED SWITCHES LETSAGO
if edge.EntityFrom != -1 {
// # 1 (2)
header = fmt.Sprintf("LET %v = (\n\tFOR x IN n%v \n", *name, edge.EntityFrom)
forEdge = fmt.Sprintf("\tFOR v, e, p IN %v..%v OUTBOUND x %s \n", edge.Depth.Min, edge.Depth.Max, edge.Type)
if edge.EntityTo != -1 {
// # 2
forSecondNode = fmt.Sprintf("\tFILTER v IN n%v \n", edge.EntityTo)
}
} else {
if edge.EntityTo != -1 {
// # 3
header = fmt.Sprintf("LET %v = (\n\tFOR x IN n%v \n", *name, edge.EntityTo)
forEdge = fmt.Sprintf("\tFOR v, e, p IN %v..%v INBOUND x %s \n", edge.Depth.Min, edge.Depth.Max, edge.Type)
} else {
// # 4
header = fmt.Sprintf("LET %v = (\n\tFOR x IN %v \n", *name, edge.Type)
footer = "\tLIMIT 100\n\tRETURN x\n)\n"
vFilterStmnt := ""
if relation.EntityTo != -1 {
// If there is a to-node, generate the filter statement
toConstraints := (*entities)[relation.EntityTo].Constraints
vFilterStmnt += *createConstraintStatements(&toConstraints, "v", false)
// Add a WITH statement if the collection of entityTo is not yet included
if (*entities)[(*relation).EntityFrom].Type != (*entities)[(*relation).EntityTo].Type {
header = fmt.Sprintf("WITH %v\n %v", (*entities)[(*relation).EntityTo].Type, header)
}
}
constraints := *createConstraints(&edge.Constraints, true, onlyEdge)
relationFilterStmnt := *createConstraintStatements(&relation.Constraints, "p", true)
footer := "\tLIMIT 1000 \nRETURN DISTINCT p )\n"
ret := header + forStatement + optionStmtn + vFilterStmnt + relationFilterStmnt + footer
return &ret
}
func createRelationLetWithOnlyToEntity(relation *relationStruct, name string, entities *[]entityStruct) *string {
header := fmt.Sprintf("LET %v = (\n\tFOR x IN n%v \n", name, relation.EntityTo)
forStatement := fmt.Sprintf("\tFOR v, e, p IN %v..%v INBOUND x %s \n", relation.Depth.Min, relation.Depth.Max, relation.Type)
// Guarantees that there is no path returned with a duplicate edge
// This way there are no cycle paths possible, TODO: more research about this needed
optionStmtn := "\tOPTIONS { uniqueEdges: \"path\" }\n"
ret := header + forEdge + forSecondNode + constraints + footer
relationFilterStmnt := *createConstraintStatements(&relation.Constraints, "p", true)
footer := "\tLIMIT 1000 \nRETURN DISTINCT p )\n"
ret := header + forStatement + optionStmtn + relationFilterStmnt + footer
return &ret
}
// WIP TODO: nodes worden nog niet meegereturned, ff googlen hoe en wat
// func createEntitylessRelationLet(edge *relationStruct, name *string, onlyEdge bool) *string {
// var (
// header string
// forEdge string
// forSecondNode string
// )
// footer := "\n\tLIMIT 100\n\tRETURN { vertices: p.vertices[*], edges: p.edges[*] }\n)\n"
// // WICKED SWITCHES LETSAGO
// if edge.EntityFrom != -1 {
// // # 1 (2) Outbound only
// header = fmt.Sprintf("LET %v = (\n\tFOR x IN n%v \n", *name, edge.EntityFrom)
// forEdge = fmt.Sprintf("\tFOR v, e, p IN %v..%v OUTBOUND x %s \n", edge.Depth.Min, edge.Depth.Max, edge.Type)
// if edge.EntityTo != -1 {
// // # 2 Node Relation Node
// forSecondNode = fmt.Sprintf("\tFILTER v IN n%v \n", edge.EntityTo)
// }
// } else {
// if edge.EntityTo != -1 {
// // # 3 Inbound only
// header = fmt.Sprintf("LET %v = (\n\tFOR x IN n%v \n", *name, edge.EntityTo)
// forEdge = fmt.Sprintf("\tFOR v, e, p IN %v..%v INBOUND x %s \n", edge.Depth.Min, edge.Depth.Max, edge.Type)
// } else {
// // # 4 Relation with constraints only
// header = fmt.Sprintf("LET %v = (\n\tFOR x IN %v \n", *name, edge.Type)
// footer = "\tLIMIT 100\n\tRETURN x\n)\n"
// }
// }
// allStatement := true
// if onlyEdge {
// allStatement = false
// }
// constraints := *createConstraintStatements(&edge.Constraints, *name, allStatement)
// ret := header + forEdge + forSecondNode + constraints + footer
// return &ret
// }
/*
#1
{
......
......@@ -8,6 +8,33 @@ import (
func TestMock(t *testing.T) {
// s := `{"Return":{"Entities":[0,1],"Relations":[0]},"Entities":[{"Type":"airports","Constraints":[{"Attribute":"country","Value":"USA","DataType":"text","MatchType":"exact"}]},{"Type":"airports","Constraints":[{"Attribute":"city","Value":"New York","DataType":"text","MatchType":"exact"},{"Attribute":"vip","Value":"true","DataType":"bool","MatchType":"exact"}]}],"Relations":[{"Type":"flights","Depth":{"min":1,"max":1},"EntityFrom":0,"EntityTo":1,"Constraints":[{"Attribute":"Month","Value":"1","DataType":"number","MatchType":"exact"},{"Attribute":"Day","Value":"15","DataType":"number","MatchType":"exact"}]}]}`
// s3 := []byte(s)
// // Convert the json byte msg to a query string
// convertQueryService := NewService()
// query, err := convertQueryService.ConvertQuery(&s3)
// if err != nil {
// errorhandler.LogError(err, "failed to parse incoming msg to query language") // TODO: send error message to client
// return
// }
// fmt.Println("Query: " + *query)
// // Make request to database
// // TODO : Generate database seperately
// // execute and retrieve result
// // convert result to general (node-link (?)) format
// requestService := request.NewService()
// result, err := requestService.SendAQLQuery(*query)
// if err != nil {
// logger.Log(err.Error())
// return // TODO: Send message in queue notifying of error
// }
// fmt.Print("QueryResult: ")
// fmt.Println(*result)
assert.True(t, true, true)
}
......@@ -81,9 +108,142 @@ func TestMock(t *testing.T) {
// }`
// s3 := []byte(s)
// j, _ := ConvertQuery(&s3)
// convertQueryService := NewService()
// j, _ := convertQueryService.ConvertQuery(&s3)
// expected := `LET n0 = (
// FOR x IN airports
// FILTER x.country == "USA"
// RETURN x
// )
// LET r0 = (
// FOR x IN n0
// FOR v, e, p IN 1..1 OUTBOUND x flights
// OPTIONS { uniqueEdges: "path" }
// FILTER v.city == "New York"
// AND v.vip == true
// FILTER p.edges[*].Month ALL == 1
// AND p.edges[*].Day ALL == 15
// LIMIT 1000
// RETURN DISTINCT p )
// LET nodes = first(RETURN UNION_DISTINCT(flatten(r0[**].vertices), [],[]))
// LET edges = first(RETURN UNION_DISTINCT(flatten(r0[**].edges), [],[]))
// RETURN {"vertices":nodes, "edges":edges }`
// assert.Equal(t, *j, expected)
// }
// func TestOnlyEntitiesQuery(t *testing.T) {
// s := `{
// "Return": {
// "Entities": [
// 0
// ],
// "Relations": []
// },
// "Entities": [
// {
// "Type": "airports",
// "Constraints": [
// {
// "Attribute": "city",
// "Value": "New York",
// "DataType": "text",
// "MatchType": "exact"
// },
// {
// "Attribute": "country",
// "Value": "USA",
// "DataType": "text",
// "MatchType": "exact"
// }
// ]
// }
// ],
// "Relations": []
// }`
// s3 := []byte(s)
// convertQueryService := NewService()
// j, _ := convertQueryService.ConvertQuery(&s3)
// expected := `LET n0 = (
// FOR x IN airports
// FILTER x.city == "New York"
// AND x.country == "USA"
// RETURN x
// )
// LET nodes = first(RETURN UNION_DISTINCT(n0,[],[]))
// LET edges = first(RETURN UNION_DISTINCT([],[]))
// RETURN {"vertices":nodes, "edges":edges }`
// assert.Equal(t, expected, *j)
// }
// func TestInboundQuery(t *testing.T) {
// s := `{
// "Return": {
// "Entities": [
// 0
// ],
// "Relations": [
// 0
// ]
// },
// "Entities": [
// {
// "Type": "airports",
// "Constraints": [
// {
// "Attribute": "city",
// "Value": "New York",
// "DataType": "text",
// "MatchType": "exact"
// }
// ]
// }
// ],
// "Relations": [
// {
// "Type": "flights",
// "Depth": {
// "min": 1,
// "max": 1
// },
// "EntityFrom": -1,
// "EntityTo": 0,
// "Constraints": [{
// "Attribute": "Day",
// "Value": "15",
// "DataType": "number",
// "MatchType": "exact"
// }]
// }
// ]
// }`
// s3 := []byte(s)
// convertQueryService := NewService()
// j, _ := convertQueryService.ConvertQuery(&s3)
// expected := `LET n0 = (
// FOR x IN airports
// FILTER x.city == "New York"
// RETURN x
// )
// LET r0 = (
// FOR x IN n0
// FOR v, e, p IN 1..1 INBOUND x flights
// OPTIONS { uniqueEdges: "path" }
// FILTER p.edges[*].Day ALL == 15
// LIMIT 1000
// RETURN DISTINCT p )
// fmt.Print(j)
// LET nodes = first(RETURN UNION_DISTINCT(flatten(r0[**].vertices), [],[]))
// LET edges = first(RETURN UNION_DISTINCT(flatten(r0[**].edges), [],[]))
// RETURN {"vertices":nodes, "edges":edges }`
// assert.True(t, true, true)
// assert.Equal(t, expected, *j)
// }
......@@ -2,7 +2,8 @@ package convertquery
import "fmt"
func createConstraints(constraints *[]constraintStruct, isRelation bool, onlyEdge bool) *string {
// createConstraintStatements generates the appropriate amount of constraint lines calling createConstraingBoolExpression
func createConstraintStatements(constraints *[]constraintStruct, name string, isRelation bool) *string {
s := ""
if len(*constraints) == 0 {
return &s
......@@ -11,14 +12,16 @@ func createConstraints(constraints *[]constraintStruct, isRelation bool, onlyEdg
newLineStatement := "\tFILTER"
for _, v := range *constraints {
s += fmt.Sprintf("%v %v \n", newLineStatement, *createConstraintLine(&v, isRelation, onlyEdge))
s += fmt.Sprintf("%v %v \n", newLineStatement, *createConstraintBoolExpression(&v, name, isRelation))
newLineStatement = "\tAND"
}
return &s
}
func createConstraintLine(constraint *constraintStruct, isRelation bool, onlyEdge bool) *string {
// createConstraintBoolExpression generates a bool expression, e.g. {name}.city == "New York".
// when isRelation is true, {name}.edges[*] ALL == "New York"
func createConstraintBoolExpression(constraint *constraintStruct, name string, isRelation bool) *string {
var (
match string
value string
......@@ -65,10 +68,10 @@ func createConstraintLine(constraint *constraintStruct, isRelation bool, onlyEdg
}
}
if isRelation && !onlyEdge {
line = fmt.Sprintf("p.edges[*].%s ALL %s %s", constraint.Attribute, match, value)
if isRelation {
line = fmt.Sprintf("%s.edges[*].%s ALL %s %s", name, constraint.Attribute, match, value)
} else {
line = fmt.Sprintf("x.%s %s %s", constraint.Attribute, match, value)
line = fmt.Sprintf("%s.%s %s %s", name, constraint.Attribute, match, value)
}
return &line
}
......@@ -34,6 +34,11 @@ type ListContainer struct {
edgeList []Document
}
type arangoResult struct {
vertices []Document
edges []Document
}
//attr interface{}
//map[1 , 2 , 3 map [ .. ]]
......@@ -66,17 +71,6 @@ func (s *Service) SendAQLQuery(AQLQuery string) (*map[string][]Document, error)
return nil, err
}
// CHANGED TO OTHER FORMAT
//fmt.Println(AQLQuery)
// query := `
// LET n0 = (
// FOR x IN airports
// FILTER x.country == 'USA'
// RETURN x
// )
// FOR n in n0
// RETURN n
// `
cursor, err := db.Query(ctx, AQLQuery, nil)
if err != nil {
log.Println("Invalid query") // handle error
......@@ -84,7 +78,7 @@ func (s *Service) SendAQLQuery(AQLQuery string) (*map[string][]Document, error)
}
defer cursor.Close()
lcontainer := ListContainer{}
listContainer := ListContainer{}
for {
var doc map[string][]interface{}
_, err := cursor.ReadDocument(ctx, &doc)
......@@ -94,73 +88,43 @@ func (s *Service) SendAQLQuery(AQLQuery string) (*map[string][]Document, error)
// handle other errors
return nil, err
}
//fmt.Printf("%s\n", doc)
//GEDACHTEGANG TIJD:
//Normaal een lijst van n0, n1. Nu kan er ook e0 bij zitten, die heeft een andere structuur
//Dus nu een returnstruct maken met een nodelist en edgelist
//Vervolgens door de keys van de doc (n0 e0 etc) loopen en een verschillende parser aanroepen die de
//returnstruct vult. Daarna de returnstruct omzetten tot een nodelist (en maybe edgelist)
//ret = parseDocToReturn() {
// for key in doc:
// if key starts with n: Nodeparsen
// if key starts with e: Edgeparsen
// return listContainer {nodelist edgelist}
//}
//
//result = parseContainerToString(ret)
parseResult(doc, &lcontainer)
parseResult(doc, &listContainer)
}
queryResult["nodes"] = lcontainer.nodeList
queryResult["edges"] = lcontainer.edgeList
queryResult["nodes"] = listContainer.nodeList
queryResult["edges"] = listContainer.edgeList
//writeJSON(queryResult)
//file, err := json.MarshalIndent(queryResult, "", " ")
return &queryResult, nil
}
// parseResult takes the result of the query and translates this to two lists: a nodelist and an edgelist, stored in a listcontainer
func parseResult(doc map[string][]interface{}, lcontainer *ListContainer) {
for k, v := range doc {
switch letter := []byte(k)[0]; letter {
case 'e':
//fmt.Println(v)
//Parsing of edges
for _, j := range v {
//fmt.Println(j)
//fmt.Printf("\n%T\n", j)
d := j.(map[string]interface{})
//fmt.Printf("\n%T\n", d["vertices"])
vert := d["vertices"].([]interface{})
edg := d["edges"].([]interface{})
lcontainer.nodeList = append(lcontainer.nodeList, parseNode(vert[0]))
lcontainer.nodeList = append(lcontainer.nodeList, parseNode(vert[1]))
lcontainer.edgeList = append(lcontainer.edgeList, parseEdge(edg[0]))
}
case 'n':
//Parsing of nodes
for _, j := range v {
//fmt.Printf("\n%T\n", j)
doc := j.(map[string]interface{})
lcontainer.nodeList = append(lcontainer.nodeList, parseNode(doc))
}
default:
//Error
fmt.Println("Empty document")
}
//Resultaat: [ { vertices : [], edges : [] } ]
func parseResult(doc map[string][]interface{}, listContainer *ListContainer) {
vertices := doc["vertices"]
edges := doc["edges"]
fmt.Println(vertices)
for _, vertex := range vertices {
vertexDoc := vertex.(map[string]interface{})
(*listContainer).nodeList = append((*listContainer).nodeList, parseNode(vertexDoc))
}
for _, edge := range edges {
edgeDoc := edge.(map[string]interface{})
(*listContainer).edgeList = append((*listContainer).edgeList, parseEdge(edgeDoc))
}
}
// parseResult takes the result of the query and translates this to two lists: a nodelist and an edgelist, stored in a listcontainer
// parseEdge parses the data of an edge to an output-friendly format
func parseEdge(d interface{}) Document {
doc := d.(map[string]interface{})
func parseEdge(d map[string]interface{}) Document {
doc := d //.(map[string]interface{})
data := make(Document)
data["_id"] = doc["_id"]
......@@ -183,23 +147,6 @@ func parseEdge(d interface{}) Document {
return data
}
// func formatToJSON(doc GeneralFormat) []Document {
// //b, err := json.Marshal(doc)
// //if err != nil {
// //handle error
// //}
// fmt.Println(doc)
// var nodeList []Document
// for _, v := range doc {
// for _, j := range v {
// nodeList = append(nodeList, parseNode(j))
// }
// }
// // fmt.Println(nodeList)
// return nodeList
// }
// writeJSON writes a json file for testing purposes
func writeJSON(queryResult map[string][]Document) {
file, _ := json.MarshalIndent(queryResult, "", " ")
......@@ -208,8 +155,8 @@ func writeJSON(queryResult map[string][]Document) {
}
// parseNode parses the data of a node to an output-friendly format
func parseNode(d interface{}) Document {
doc := d.(map[string]interface{})
func parseNode(d map[string]interface{}) Document {
doc := d //.(map[string]interface{})
data := make(Document)
data["_id"] = doc["_id"]
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment