Skip to content
Snippets Groups Projects
Commit 7265311a authored by Heijden,T.A.J. van der (Thijs)'s avatar Heijden,T.A.J. van der (Thijs)
Browse files

Merge branch 'develop' into 'main'

End of SWP 2 Merge

See merge request !4
parents 857619aa 82d6956b
Branches
No related tags found
1 merge request!4End of SWP 2 Merge
Pipeline #115047 failed
Showing
with 5307 additions and 2277 deletions
*.out
*.html
coverage*
*.txt
\ No newline at end of file
#This program has been developed by students from the bachelor Computer Science at Utrecht University within the Software Project course.
#© Copyright Utrecht University (Department of Information and Computing Sciences)
image: golang:1.16
stages:
- test
unit_tests:
stage: test
script:
- make dep
- make test
- gocover-cobertura < coverage.txt > coverage.xml
artifacts:
reports:
cobertura: coverage.xml
coverage:
stage: test
script:
- make coverage
after_script:
- mkdir $CI_COMMIT_BRANCH
- cp cover.html $CI_COMMIT_BRANCH
- mv $CI_COMMIT_BRANCH/cover.html $CI_COMMIT_BRANCH/index.html
- if [[ $CI_COMMIT_BRANCH = "develop" || $CI_COMMIT_BRANCH = "main" ]]; then COVERAGE_PATH=""; else COVERAGE_PATH="features"; fi
# install openssh client and add ssh keys
- apt-get install openssh-client curl -y >/dev/null
- mkdir ~/.ssh/
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- ssh-add ~/.ssh/id_rsa
- echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
- ssh -fN -L 1234:science-vs260.science.uu.nl:22 sivan@up.science.uu.nl
- scp -r -o StrictHostKeyChecking=no -P 1234 -i ~/.ssh/id_rsa $CI_COMMIT_BRANCH root@localhost:/datadisk/documentation-coverage/home/backend/$CI_PROJECT_NAME/$COVERAGE_PATH
artifacts:
untracked: false
expire_in: 30 days
paths:
- cover.html
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${fileDirname}"
}
]
}
\ No newline at end of file
Makefile 0 → 100644
.PHONY: dep test
dep: ## Get the dependencies
@go get -a ./...
@go get -u github.com/boumenot/gocover-cobertura
test: dep ## Run unittests
@go test -coverpkg=./aql -coverprofile=aql_coverage.txt -covermode count ./...
@cat aql_coverage.txt > coverage.txt
@go test -coverpkg=./sparql -coverprofile=sparql_coverage.txt -covermode count ./...
@tail -n +2 ./sparql_coverage.txt >> coverage.txt
@go test -coverpkg=./entity -coverprofile=entity_coverage.txt -covermode count ./...
@tail -n +2 ./entity_coverage.txt >> coverage.txt
coverage: dep
@go test -v -coverpkg=./aql -coverprofile=aql_cover.out ./...
@cat aql_cover.out > cover.out
@go test -v -coverpkg=./sparql -coverprofile=sparql_cover.out ./...
@tail -n +2 ./sparql_cover.out >> cover.out
@go test -v -coverpkg=./entity -coverprofile=entity_cover.out ./...
@tail -n +2 ./entity_cover.out >> cover.out
@go tool cover -func cover.out | grep total
@go tool cover -html=cover.out -o cover.html
\ No newline at end of file
[![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
<div align='center'>
<img src="https://git.science.uu.nl/GraphPolaris/frontend/-/raw/develop/src/presentation/view/navbar/logogp.png" align="center" width="150" alt="Project icon">
</div>
# Query Converter
This is the query-conversion package. It holds the code that translates the JSON of the visual query to the selected database language. There are currently two implementations, one for the Arango Query Language and one for Cypher.
This is the query-conversion package. It holds the code that translates the JSON of the visual query to the selected database language. There are currently three implementations, one for the Arango Query Language, one for Cypher and finally one for SPARQL.
## Creating a new AQL converter
```go
import "git.science.uu.nl/graphpolaris/query-conversion"
queryservice := aql.NewService()
```
## Creating a new converter
## Creating a new Cypher converter
```go
import "git.science.uu.nl/graphpolaris/query-conversion"
queryservice := cypher.NewService()
```
## Creating a new SPARQL converter
```go
import "git.science.uu.nl/graphpolaris/query-conversion"
queryservice := sparql.NewService()
```
## Converting a query
A query can be made by providing the JSON of the visual query and feeding it into the `ConvertQuery` function.
......@@ -21,3 +41,9 @@ import "git.science.uu.nl/graphpolaris/query-conversion"
mockService := NewMockService()
```
### Dependencies
This service depends on RabbitMQ
### Testing and Coverage
To test the `make test` command can be used. It will run every test in the repo. To get the code coverage the command `make coverage` can be used. This command will run all tests and display total coverage over all files. It will also generate a html file which displays exactly which lines have been covered.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
package aql
import (
"fmt"
"git.science.uu.nl/graphpolaris/query-conversion/entity"
)
/* createConstraintStatements generates the appropriate amount of constraint lines calling createConstraingBoolExpression
Parameters: constraints is a list of constraintStructs that specify the constraints of a node or relation,
name is the id of the corresponding relation/node,
isRelation is a boolean specifying if this constraint comes from a node or relation
Return: a string containing a FILTER-statement with all the constraints
*/
func createConstraintStatements(constraints *[]entity.QueryConstraintStruct, name string, isRelation bool) *string {
s := ""
if len(*constraints) == 0 {
return &s
}
newLineStatement := "\tFILTER"
for _, v := range *constraints {
s += fmt.Sprintf("%v %v \n", newLineStatement, *createConstraintBoolExpression(&v, name, isRelation))
newLineStatement = "\tAND"
}
return &s
}
/* createConstraintBoolExpression generates a single boolean expression,
e.g. {name}.city == "New York".
Parameters: constraint is a single constraint of a node or relation,
name is the id of the corresponding relation/node,
isRelation is a boolean specifying if this constraint comes from a node or relation, that changes the structure of the expression
Return: a string containing an boolean expression of a single constraint
*/
func createConstraintBoolExpression(constraint *entity.QueryConstraintStruct, name string, isRelation bool) *string {
var (
match string
value string
line string
)
// Constraint datatypes back end
// string MatchTypes: EQ/NEQ/contains/excludes
// int MatchTypes: EQ/NEQ/GT/LT/GET/LET
// bool MatchTypes: EQ/NEQ
switch constraint.DataType {
case "string":
value = fmt.Sprintf("\"%s\"", constraint.Value)
switch constraint.MatchType {
case "NEQ":
match = "!="
case "contains":
match = "LIKE"
value = fmt.Sprintf("\"%%%s%%\"", constraint.Value)
case "excludes":
match = "NOT LIKE"
value = fmt.Sprintf("\"%%%s%%\"", constraint.Value)
default: //EQ
match = "=="
}
case "int":
value = constraint.Value
switch constraint.MatchType {
case "NEQ":
match = "!="
case "GT":
match = ">"
case "LT":
match = "<"
case "GET":
match = ">="
case "LET":
match = "<="
default: //EQ
match = "=="
}
default: /*bool*/
value = constraint.Value
switch constraint.MatchType {
case "NEQ":
match = "!="
default: //EQ
match = "=="
}
}
if isRelation {
line = fmt.Sprintf("%s.edges[*].%s ALL %s %s", name, constraint.Attribute, match, value)
} else {
line = fmt.Sprintf("%s.%s %s %s", name, constraint.Attribute, match, value)
}
return &line
}
package aql
import (
"git.science.uu.nl/graphpolaris/query-conversion/entity"
)
// We use consts to define string to prevent typos
const ENTITYSTRING = "entity"
const RELATIONSTRING = "relation"
const GROUPBYSTRING = "groupBy"
var relationdone map[int]bool
func createHierarchy(JSONQuery *entity.IncomingQueryJSON, entityMap map[int]int, relationMap map[int]int) ([]entity.Tree, entity.QueryEntityStruct) {
var treeList []entity.Tree
relationdone = make(map[int]bool)
topNode := getTopNode(JSONQuery)
topTreeSelfTriple := getTripleFromNode(JSONQuery, topNode, entityMap, relationMap)
topTree := entity.Tree{
Self: topTreeSelfTriple,
Parent: -1,
Children: []int{},
}
treeList = append(treeList, topTree)
treeListIndex := len(treeList) - 1
treeList = getChildrenFromTree(JSONQuery, treeList, treeListIndex, 0, entityMap)
return treeList, topNode
}
/*
Get a entity that has only 1 relation attached an return its index
*/
func getTopNode(JSONQuery *entity.IncomingQueryJSON) entity.QueryEntityStruct {
indexOfNodeToReturn := -1
for i, node := range JSONQuery.Entities {
connectionCount := 0
for _, relation := range JSONQuery.Relations {
if (relation.FromType == ENTITYSTRING && relation.FromID == node.ID) || (relation.ToType == ENTITYSTRING && relation.ToID == node.ID) {
connectionCount++
}
}
if connectionCount == 1 {
indexOfNodeToReturn = i
return JSONQuery.Entities[indexOfNodeToReturn]
}
}
return JSONQuery.Entities[indexOfNodeToReturn]
}
func getTripleFromNode(JSONQuery *entity.IncomingQueryJSON, node entity.QueryEntityStruct, entityMap map[int]int, relationMap map[int]int) entity.Triple {
var tripleToReturn entity.Triple
for _, relation := range JSONQuery.Relations {
// The initial node was our From so we set the Triple accordingly
// TODO
// If the To is not an entity we might have to do something different
if (relation.FromType == ENTITYSTRING && relation.FromID == node.ID) && relation.ToType == ENTITYSTRING {
tripleToReturn.FromNode = node
tripleToReturn.Rel = relation
tripleToReturn.ToNode = JSONQuery.Entities[entityMap[relation.ToID]]
} else if (relation.ToType == ENTITYSTRING && relation.ToID == node.ID) && relation.FromType == ENTITYSTRING {
tripleToReturn.FromNode = JSONQuery.Entities[entityMap[relation.FromID]]
tripleToReturn.Rel = relation
tripleToReturn.ToNode = node
}
}
relationdone[relationMap[tripleToReturn.Rel.ID]] = true
return tripleToReturn
}
func getTripleFromRelation(JSONQuery *entity.IncomingQueryJSON, relation entity.QueryRelationStruct, entityMap map[int]int) entity.Triple {
var tripleToReturn entity.Triple
tripleToReturn.FromNode = JSONQuery.Entities[entityMap[relation.FromID]]
tripleToReturn.Rel = relation
tripleToReturn.ToNode = JSONQuery.Entities[entityMap[relation.ToID]]
return tripleToReturn
}
func getChildrenFromTree(JSONQuery *entity.IncomingQueryJSON, treeList []entity.Tree, treeListIndex int, parentIndex int, entityMap map[int]int) []entity.Tree {
var childRelationTriples []entity.Triple
for i, relation := range JSONQuery.Relations {
// We found a relation that is not our parent relation so we can check if it matches on of our nodes
// If it matches one of the nodes we can add it
if _, ok := relationdone[i]; !ok {
if relation.FromType == ENTITYSTRING && relation.FromID == treeList[parentIndex].Self.FromNode.ID {
triple := getTripleFromRelation(JSONQuery, relation, entityMap)
childRelationTriples = append(childRelationTriples, triple)
relationdone[i] = true
} else if relation.ToType == ENTITYSTRING && relation.ToID == treeList[parentIndex].Self.ToNode.ID {
triple := getTripleFromRelation(JSONQuery, relation, entityMap)
childRelationTriples = append(childRelationTriples, triple)
relationdone[i] = true
}
}
}
// We now have all our children, so we can now make those trees and find their children
// We can now also add the indices to the list of children from the tree calling this function
if len(childRelationTriples) != 0 {
for _, triple := range childRelationTriples {
childTree := entity.Tree{
Self: triple,
Parent: parentIndex,
Children: []int{},
}
treeList = append(treeList, childTree)
// We get the new treeListIndex, which we can now add to the list of children from the tree calling this function
treeListIndex = len(treeList) - 1
treeList[parentIndex].Children = append(treeList[parentIndex].Children, treeListIndex)
treeList = getChildrenFromTree(JSONQuery, treeList, treeListIndex, treeListIndex, entityMap)
}
}
return treeList
}
package aql
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.science.uu.nl/graphpolaris/query-conversion/entity"
"github.com/stretchr/testify/assert"
)
func TestHierarchyBasic(t *testing.T) {
// Setup for test
// Create query conversion service
query := []byte(`{
"return": {
"entities": [
0,
1,
2,
3,
4
],
"relations": [
0,
1,
2,
3
]
},
"entities": [
{
"name": "parliament",
"ID": 0,
"constraints": [
{
"attribute": "name",
"value": "Geert",
"dataType": "string",
"matchType": "CONTAINS"
}
]
},
{
"name": "commissions",
"ID": 1,
"constraints": []
},
{
"name": "parliament",
"ID": 2,
"constraints": []
},
{
"name": "parties",
"ID": 3,
"constraints": [
{
"attribute": "seats",
"value": "10",
"dataType": "int",
"matchType": "LT"
}
]
},
{
"name": "resolutions",
"ID": 4,
"constraints": [
{
"attribute": "date",
"value": "mei",
"dataType": "string",
"matchType": "CONTAINS"
}
]
}
],
"groupBys": [],
"relations": [
{
"ID": 0,
"name": "part_of",
"depth": {
"min": 1,
"max": 1
},
"fromType": "entity",
"fromId": 0,
"toType": "entity",
"toID": 1,
"constraints": []
},
{
"ID": 1,
"name": "part_of",
"depth": {
"min": 1,
"max": 1
},
"fromType": "entity",
"fromId": 2,
"toType": "entity",
"toID": 1,
"constraints": []
},
{
"ID": 2,
"name": "member_of",
"depth": {
"min": 1,
"max": 1
},
"fromType": "entity",
"fromId": 2,
"toType": "entity",
"toID": 3,
"constraints": []
},
{
"ID": 3,
"name": "submits",
"depth": {
"min": 1,
"max": 1
},
"fromType": "entity",
"fromId": 2,
"toType": "entity",
"toID": 4,
"constraints": []
}
],
"limit": 5000
}
`)
// Unmarshall the incoming message into an IncomingJSONQuery object
var JSONQuery entity.IncomingQueryJSON
json.Unmarshal(query, &JSONQuery)
// Get the hierarchy and turn it into JSON so we can turn it into strings later
entityMap, relationMap, _ := entity.FixIndices(&JSONQuery)
treeList, topNode := createHierarchy(&JSONQuery, entityMap, relationMap)
jsonTopNode, err := json.Marshal(topNode)
if err != nil {
fmt.Println("Marshalling went wrong")
}
jsonTreeList, err := json.Marshal(treeList)
if err != nil {
fmt.Println("Marshalling went wrong")
}
// These are the expected (correct) outputs
correctTopNode := []byte(`{"ID":0,"Name":"parliament","Constraints":[{"Attribute":"name","Value":"Geert","DataType":"string","MatchType":"CONTAINS","InID":0,"InType":""}]}`)
correctTreeList := []byte(`[
{
"Self": {
"FromNode": {
"ID": 0,
"Name": "parliament",
"Constraints": [
{
"Attribute": "name",
"Value": "Geert",
"DataType": "string",
"MatchType": "CONTAINS",
"InID": 0,
"InType": ""
}
]
},
"Rel": {
"ID": 0,
"Name": "part_of",
"FromType": "entity",
"FromID": 0,
"ToType": "entity",
"ToID": 1,
"Depth": {
"Min": 1,
"Max": 1
},
"Constraints": []
},
"ToNode": {
"ID": 1,
"Name": "commissions",
"Constraints": []
}
},
"Parent": -1,
"Children": [
1
]
},
{
"Self": {
"FromNode": {
"ID": 2,
"Name": "parliament",
"Constraints": []
},
"Rel": {
"ID": 1,
"Name": "part_of",
"FromType": "entity",
"FromID": 2,
"ToType": "entity",
"ToID": 1,
"Depth": {
"Min": 1,
"Max": 1
},
"Constraints": []
},
"ToNode": {
"ID": 1,
"Name": "commissions",
"Constraints": []
}
},
"Parent": 0,
"Children": [
2,
3
]
},
{
"Self": {
"FromNode": {
"ID": 2,
"Name": "parliament",
"Constraints": []
},
"Rel": {
"ID": 2,
"Name": "member_of",
"FromType": "entity",
"FromID": 2,
"ToType": "entity",
"ToID": 3,
"Depth": {
"Min": 1,
"Max": 1
},
"Constraints": []
},
"ToNode": {
"ID": 3,
"Name": "parties",
"Constraints": [
{
"Attribute": "seats",
"Value": "10",
"DataType": "int",
"MatchType": "LT",
"InID": 0,
"InType": ""
}
]
}
},
"Parent": 1,
"Children": []
},
{
"Self": {
"FromNode": {
"ID": 2,
"Name": "parliament",
"Constraints": []
},
"Rel": {
"ID": 3,
"Name": "submits",
"FromType": "entity",
"FromID": 2,
"ToType": "entity",
"ToID": 4,
"Depth": {
"Min": 1,
"Max": 1
},
"Constraints": []
},
"ToNode": {
"ID": 4,
"Name": "resolutions",
"Constraints": [
{
"Attribute": "date",
"Value": "mei",
"DataType": "string",
"MatchType": "CONTAINS",
"InID": 0,
"InType": ""
}
]
}
},
"Parent": 1,
"Children": []
}
]`)
// Clean up the input and expected results
cleanedTopNode := strings.ReplaceAll(string(jsonTopNode), "\n", "")
cleanedTopNode = strings.ReplaceAll(cleanedTopNode, "\t", "")
cleanedTopNode = strings.ReplaceAll(cleanedTopNode, " ", "")
cleanedTreeList := strings.ReplaceAll(string(jsonTreeList), "\n", "")
cleanedTreeList = strings.ReplaceAll(cleanedTreeList, "\t", "")
cleanedTreeList = strings.ReplaceAll(cleanedTreeList, " ", "")
cleanedCorrectTopNode := strings.ReplaceAll(string(correctTopNode), "\n", "")
cleanedCorrectTopNode = strings.ReplaceAll(cleanedCorrectTopNode, "\t", "")
cleanedCorrectTopNode = strings.ReplaceAll(cleanedCorrectTopNode, " ", "")
cleanedCorrectTreeList := strings.ReplaceAll(string(correctTreeList), "\n", "")
cleanedCorrectTreeList = strings.ReplaceAll(cleanedCorrectTreeList, "\t", "")
cleanedCorrectTreeList = strings.ReplaceAll(cleanedCorrectTreeList, " ", "")
assert.Equal(t, cleanedCorrectTopNode, cleanedTopNode)
assert.Equal(t, cleanedCorrectTreeList, cleanedTreeList)
}
/*
This program has been developed by students from the bachelor Computer Science at Utrecht University within the Software Project course.
© Copyright Utrecht University (Department of Information and Computing Sciences)
*/
package aql
// Service implements the QueryConverter interface (in the query service)
......
......
This diff is collapsed.
/*
This program has been developed by students from the bachelor Computer Science at Utrecht University within the Software Project course.
© Copyright Utrecht University(Department of Information and Computing Sciences)
*/
package cypher
import (
......@@ -96,7 +101,12 @@ func checkForQueryCluster(JSONQuery *entity.IncomingQueryJSON) (*entity.Incoming
for _, ent := range JSONQuery.Entities {
self := fmt.Sprintf("e%v", ent.ID)
for _, con := range ent.Constraints {
for i, con := range ent.Constraints {
if con.InType == "" {
ent.Constraints[i].InID = -1
continue
}
if con.InID != -1 {
in := fmt.Sprintf("%v%v", string(con.InType[0]), con.InID)
......@@ -286,7 +296,7 @@ func checkNoDeadEnds(JSONQuery *entity.IncomingQueryJSON) (bool, error) {
}
for _, cons := range ent.Constraints {
if cons.InID == -1 {
if cons.InID == -1 || cons.InType == "" {
continue
}
......
......
/*
This program has been developed by students from the bachelor Computer Science at Utrecht University within the Software Project course.
© Copyright Utrecht University(Department of Information and Computing Sciences)
*/
package cypher
import (
......@@ -128,7 +133,7 @@ func createReturnStatement(JSONQuery *entity.IncomingQueryJSON, parts entity.Que
}
idstr += "L"
}
returnlist = append(returnlist, fmt.Sprintf("eg%v", idstr)) // HIER GAAT NOG WAT MIS (TODO)
returnlist = append(returnlist, fmt.Sprintf("eg%v", idstr))
}
}
......@@ -146,7 +151,7 @@ func createReturnStatement(JSONQuery *entity.IncomingQueryJSON, parts entity.Que
}
idstr += "L"
}
returnlist = append(returnlist, fmt.Sprintf("eg%v", idstr)) // Hier vgm ook, ffkes kijken hoe en wat het zit met relaties aan een groupby
returnlist = append(returnlist, fmt.Sprintf("eg%v", idstr))
}
}
} else if part.QType == "entity" {
......@@ -272,6 +277,7 @@ func createInCypher(JSONQuery *entity.IncomingQueryJSON, part entity.QueryPart)
}
// createRelationCypher takes the json and a query part, finds the necessary entities and converts it into cypher
func createRelationCypher(JSONQuery *entity.IncomingQueryJSON, part entity.QueryPart) (*string, error) {
rel := JSONQuery.FindR(part.QID)
......
......
/*
This program has been developed by students from the bachelor Computer Science at Utrecht University within the Software Project course.
© Copyright Utrecht University(Department of Information and Computing Sciences)
*/
package cypher
import (
......
......
/*
This program has been developed by students from the bachelor Computer Science at Utrecht University within the Software Project course.
© Copyright Utrecht University(Department of Information and Computing Sciences)
*/
package cypher
import (
......
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment