Updated objectstore to use less memory

Added status messages for queries in progress.

.gitignore

 builds/
 cover.html
 cover.out
-coverage.txt
\ No newline at end of file
+coverage.txt
+.idea
+.DS_Store
\ No newline at end of file

go.mod

@@ -5,8 +5,8 @@ go 1.16
 require (
 	git.science.uu.nl/datastrophe/broker v0.0.0-20210516094125-abbeaf96fd58
 	git.science.uu.nl/datastrophe/keyvaluestore v0.0.0-20210527103454-a4f448f11f28
-	git.science.uu.nl/datastrophe/objectstore v0.0.0-20210527105610-6eaa83999d9e
-	git.science.uu.nl/datastrophe/query-conversion v0.0.0-20210518093948-06ff65cdf577
+	git.science.uu.nl/datastrophe/objectstore v0.0.0-20210603160015-4cbd339b979c
+	git.science.uu.nl/datastrophe/query-conversion v0.0.0-20210601141629-c90342e61f80
 	git.science.uu.nl/datastrophe/query-execution v0.0.0-20210518112715-043c7046dc7b
 	github.com/stretchr/testify v1.7.0
 	github.com/thijsheijden/alice v0.1.18

go.sum

@@ -7,8 +7,12 @@ git.science.uu.nl/datastrophe/objectstore v0.0.0-20210521093716-251df4af345f h1:
 git.science.uu.nl/datastrophe/objectstore v0.0.0-20210521093716-251df4af345f/go.mod h1:O3nZ8zKqPIPQmehc0jdgAWx2pvkgcuoVQn8Zqa8FH0s=
 git.science.uu.nl/datastrophe/objectstore v0.0.0-20210527105610-6eaa83999d9e h1:qfhFPgvgRpbmM1EZCRDRaTa4r32Rd9hdyIcFGpjs+Zc=
 git.science.uu.nl/datastrophe/objectstore v0.0.0-20210527105610-6eaa83999d9e/go.mod h1:odagvf0EkoQ5xZmEVWDqaLv1iLjpQUICfG8ZZTXiWtw=
+git.science.uu.nl/datastrophe/objectstore v0.0.0-20210603160015-4cbd339b979c h1:tU4ZVOCG27CA5aRLJJmQcaQZWdAv5YFAjLpHJmlia/w=
+git.science.uu.nl/datastrophe/objectstore v0.0.0-20210603160015-4cbd339b979c/go.mod h1:XhvvkmaqyziV5dvB7rF9/Bk+JI7BvZEnwh3hgkG8hKU=
 git.science.uu.nl/datastrophe/query-conversion v0.0.0-20210518093948-06ff65cdf577 h1:Xd9hpnnRhYWdAXzLFviA6fYWnQoFD/4NtzmEC7Ntc/U=
 git.science.uu.nl/datastrophe/query-conversion v0.0.0-20210518093948-06ff65cdf577/go.mod h1:6rvalwekoukmVu3SbWmZkj8wBZEm34wDbA4Ilxcb+jw=
+git.science.uu.nl/datastrophe/query-conversion v0.0.0-20210601141629-c90342e61f80 h1:GLGmhO6bBYd98RCd8DfykSzlIw2nDiDcbgrnHB8Ki60=
+git.science.uu.nl/datastrophe/query-conversion v0.0.0-20210601141629-c90342e61f80/go.mod h1:6rvalwekoukmVu3SbWmZkj8wBZEm34wDbA4Ilxcb+jw=
 git.science.uu.nl/datastrophe/query-execution v0.0.0-20210518112715-043c7046dc7b h1:mEuRIwUfpblTbk9DMkGwlD66bV7VSyCGXW+lgqGCoF8=
 git.science.uu.nl/datastrophe/query-execution v0.0.0-20210518112715-043c7046dc7b/go.mod h1:4YfKaCyadGlmAVbKOa6J2E2H56T7+RTF7xVCmcTCwmM=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

internal/entity/messageStruct.go

@@ -5,3 +5,13 @@ type MessageStruct struct {
 	Type  string      `json:"type"`
 	Value interface{} `json:"value"`
 }
+
+type StatusMessageStruct struct {
+	Type  string `json:"type"`
+	Value Status `json:"value"`
+}
+
+type Status struct {
+	Status  string `json:"status"`
+	QueryID string `json:"queryID"`
+}

internal/usecases/consume/handlemessage.go

@@ -33,7 +33,7 @@ func (s *Service) HandleMessage(msg *broker.Message) {
 		queryID = ""
 	}
 
-	s.sendStatus("Received", &sessionID)
+	s.sendStatus("Received", &sessionID, &queryID)
 
 	// Unmarshall the incoming message into an IncomingJSONQuery object
 	JSONQuery, err := query.UnmarshalJSON(&msg.Body)
@@ -58,7 +58,7 @@ func (s *Service) HandleMessage(msg *broker.Message) {
 		return
 	}
 
-	s.sendStatus("Translating", &sessionID)
+	s.sendStatus("Translating", &sessionID, &queryID)
 
 	// Convert the json byte msg to a query string
 	query, err := s.queryConverter.ConvertQuery(JSONQuery)
@@ -89,7 +89,7 @@ func (s *Service) HandleMessage(msg *broker.Message) {
 	}
 	s.producer.PublishMessage(&translationMsgBytes, &sessionID) // TODO: should this be a go routine?
 
-	s.sendStatus("Getting database credentials", &sessionID)
+	s.sendStatus("Getting database credentials", &sessionID, &queryID)
 
 	// Fetch database credentials from the user service
 	databaseInfo, err := s.databaseInfoService.GetDatabaseInfo(&clientID, &JSONQuery.DatabaseName)
@@ -103,7 +103,7 @@ func (s *Service) HandleMessage(msg *broker.Message) {
 		return
 	}
 
-	s.sendStatus("Processing on database", &sessionID)
+	s.sendStatus("Processing on database", &sessionID, &queryID)
 
 	// execute and retrieve result
 	// convert result to general (node-link) format
@@ -118,7 +118,7 @@ func (s *Service) HandleMessage(msg *broker.Message) {
 		return
 	}
 
-	s.sendStatus("Completed", &sessionID)
+	s.sendStatus("Completed", &sessionID, &queryID)
 
 	// Add type indicator to result from database
 	var res interface{}
@@ -140,14 +140,17 @@ func (s *Service) HandleMessage(msg *broker.Message) {
 	if queryID != "" {
 		cacheQueryResultContext, cancelCacheQueryResult := context.WithTimeout(context.Background(), time.Second*20)
 		defer cancelCacheQueryResult()
-		s.objectStore.Put(cacheQueryResultContext, "cached-queries", fmt.Sprintf("%s-%s", sessionID, queryID), bytes.NewReader(resultMsgBytes))
+		s.objectStore.Put(cacheQueryResultContext, "cached-queries", fmt.Sprintf("%s-%s", sessionID, queryID), int64(len(resultMsgBytes)), bytes.NewReader(resultMsgBytes))
 	}
 }
 
-func (s *Service) sendStatus(status string, sessionID *string) {
-	msg := entity.MessageStruct{
-		Type:  "query_status_update",
-		Value: status,
+func (s *Service) sendStatus(status string, sessionID *string, queryID *string) {
+	msg := entity.StatusMessageStruct{
+		Type: "query_status_update",
+		Value: entity.Status{
+			Status:  status,
+			QueryID: *queryID,
+		},
 	}
 
 	msgBytes, err := json.Marshal(msg)