From eee3239fe882ea52795f132f883338e5afff6c7e Mon Sep 17 00:00:00 2001
From: Leonardo <leomilho@gmail.com>
Date: Mon, 10 Mar 2025 10:27:13 +0100
Subject: [PATCH] feat: next version of postgres
---
src/queryExecution/cypher/converter/logic.ts | 10 +-
src/queryExecution/sql/convertLogicToSQL.ts | 158 ++++++++
.../sql/queryConverterSql copy.ts | 158 ++++++++
.../sql/queryConverterSql.test.ts | 2 +-
src/queryExecution/sql/queryConverterSql.ts | 362 ++++++++++++------
src/queryExecution/sql/queryResultParser.ts | 121 ------
src/readers/services/sqlService.ts | 19 +-
7 files changed, 575 insertions(+), 255 deletions(-)
create mode 100644 src/queryExecution/sql/convertLogicToSQL.ts
create mode 100644 src/queryExecution/sql/queryConverterSql copy.ts
delete mode 100644 src/queryExecution/sql/queryResultParser.ts
diff --git a/src/queryExecution/cypher/converter/logic.ts b/src/queryExecution/cypher/converter/logic.ts
index 4ef8a20..3010539 100644
--- a/src/queryExecution/cypher/converter/logic.ts
+++ b/src/queryExecution/cypher/converter/logic.ts
@@ -27,12 +27,16 @@ export function createWhereLogic(
return { logic: newWhereLogic, where: whereLogic };
}
-export function extractLogicCypher(logicQuery: AnyStatement, cacheData: QueryCacheData): { logic: string; where: Set<string> } {
+export function extractLogicCypher(
+ logicQuery: AnyStatement | AnyStatement[],
+ cacheData: QueryCacheData,
+): { logic: string; where: Set<string> } {
switch (typeof logicQuery) {
case 'object':
if (Array.isArray(logicQuery)) {
- let op = logicQuery[0].replace('_', '').toLowerCase();
- let right = logicQuery?.[2];
+ logicQuery = logicQuery as AnyStatement[];
+ let op = (logicQuery[0] as string).replace('_', '').toLowerCase();
+ const right = logicQuery?.[2];
const { logic: left, where: whereLogic } = extractLogicCypher(logicQuery[1], cacheData);
switch (op) {
diff --git a/src/queryExecution/sql/convertLogicToSQL.ts b/src/queryExecution/sql/convertLogicToSQL.ts
new file mode 100644
index 0000000..055910f
--- /dev/null
+++ b/src/queryExecution/sql/convertLogicToSQL.ts
@@ -0,0 +1,158 @@
+import { StringFilterTypes, type AnyStatement } from 'ts-common/src/model/query/logic/general';
+import { log } from 'ts-common/src/logger/logger';
+import type { NodeStruct } from 'ts-common';
+
+export type SQLCacheData = {
+ nodes: NodeStruct[];
+};
+
+export function createWhereLogic(
+ op: string,
+ left: string,
+ whereLogic: Set<string>,
+ cacheData: SQLCacheData,
+): { logic: string; where: Set<string> } {
+ const newWhereLogic = `${left.replace('.', '_')}_${op}`;
+ const tableName = left.split('.')[0];
+
+ whereLogic.add(`${op}(${left}) AS ${newWhereLogic}`);
+ for (const entity of cacheData.nodes) {
+ if (entity.id !== tableName && entity.id) {
+ whereLogic.add(entity.id);
+ }
+ }
+
+ return { logic: newWhereLogic, where: whereLogic };
+}
+
+export function extractLogicSQL(logicQuery: AnyStatement | AnyStatement[], cacheData: SQLCacheData): { logic: string; where: Set<string> } {
+ switch (typeof logicQuery) {
+ case 'object':
+ if (Array.isArray(logicQuery)) {
+ logicQuery = logicQuery as AnyStatement[];
+ let op = (logicQuery[0] as string).replace('_', '').toLowerCase();
+ const right = logicQuery?.[2];
+ const { logic: left, where: whereLogic } = extractLogicSQL(logicQuery[1], cacheData);
+
+ if (left === '') {
+ return { logic: '', where: whereLogic };
+ }
+
+ switch (op) {
+ case '!=':
+ op = '!=';
+ break;
+ case '==':
+ op = '=';
+ break;
+ case 'like':
+ return {
+ logic: `${left} LIKE '%${(right as string).replace(/['"]/g, '')}%'`,
+ where: whereLogic,
+ };
+ case 'in':
+ case 'present in list':
+ if (typeof right === 'string') {
+ return {
+ logic: `${left} IN (${right
+ .replace(/"/g, "'")
+ .split(',')
+ .map((r: string) => r.trim())
+ .join(', ')})`,
+ where: whereLogic,
+ };
+ }
+ break;
+ case 'not in':
+ case 'not in list':
+ if (typeof right === 'string') {
+ return {
+ logic: `${left} NOT IN (${right
+ .replace(/"/g, "'")
+ .split(',')
+ .map((r: string) => r.trim())
+ .join(', ')})`,
+ where: whereLogic,
+ };
+ }
+ break;
+ case 'isempty':
+ return { logic: `(${left} IS NULL OR ${left} = '')`, where: whereLogic };
+ case 'isnotempty':
+ return { logic: `(${left} IS NOT NULL AND ${left} <> '')`, where: whereLogic };
+ case 'lower':
+ return { logic: `LOWER(${left})`, where: whereLogic };
+ case 'upper':
+ return { logic: `UPPER(${left})`, where: whereLogic };
+ case 'avg':
+ return {
+ logic: `(SELECT AVG(${left}) FROM ${left.split('.')[0]})`,
+ where: whereLogic,
+ };
+ case 'count':
+ return {
+ logic: `(SELECT COUNT(${left}) FROM ${left.split('.')[0]})`,
+ where: whereLogic,
+ };
+ case 'max':
+ return {
+ logic: `(SELECT MAX(${left}) FROM ${left.split('.')[0]})`,
+ where: whereLogic,
+ };
+ case 'min':
+ return {
+ logic: `(SELECT MIN(${left}) FROM ${left.split('.')[0]})`,
+ where: whereLogic,
+ };
+ case 'sum':
+ return {
+ logic: `(SELECT SUM(${left}) FROM ${left.split('.')[0]})`,
+ where: whereLogic,
+ };
+ case 'and':
+ if (logicQuery.length > 2) {
+ const conditions = logicQuery.slice(1).map(condition => {
+ const { logic: condLogic } = extractLogicSQL(condition, cacheData);
+ return condLogic;
+ });
+ return { logic: `(${conditions.join(' AND ')})`, where: whereLogic };
+ }
+ break;
+ case 'or':
+ if (logicQuery.length > 2) {
+ const conditions = logicQuery.slice(1).map(condition => {
+ const { logic: condLogic } = extractLogicSQL(condition, cacheData);
+ return condLogic;
+ });
+ return { logic: `(${conditions.join(' OR ')})`, where: whereLogic };
+ }
+ break;
+ }
+ if (logicQuery.length > 2 && logicQuery[2] !== undefined) {
+ const { logic: rightLogic, where: whereLogicRight } = extractLogicSQL(logicQuery[2], cacheData);
+
+ for (const where of whereLogicRight) {
+ whereLogic.add(where);
+ }
+
+ return { logic: `(${left} ${op} ${rightLogic})`, where: whereLogic };
+ }
+ return { logic: `(${op} ${left})`, where: whereLogic };
+ }
+ return { logic: logicQuery, where: new Set() };
+ case 'string':
+ // Handle string literals and field references
+ if (logicQuery.startsWith('"') && logicQuery.endsWith('"')) {
+ return { logic: logicQuery.replace(/"/g, "'"), where: new Set() };
+ }
+ if (logicQuery.startsWith('@')) {
+ return { logic: logicQuery.replace('@', ''), where: new Set() };
+ } else {
+ return { logic: `'${logicQuery}'`, where: new Set() };
+ }
+ case 'number':
+ return { logic: logicQuery.toString(), where: new Set() };
+ default:
+ return { logic: logicQuery as any, where: new Set() };
+ }
+}
diff --git a/src/queryExecution/sql/queryConverterSql copy.ts b/src/queryExecution/sql/queryConverterSql copy.ts
new file mode 100644
index 0000000..c370a1e
--- /dev/null
+++ b/src/queryExecution/sql/queryConverterSql copy.ts
@@ -0,0 +1,158 @@
+import type { BackendQueryFormat, NodeStruct } from 'ts-common';
+import type { QueryText } from '../model';
+
+type Logic = any;
+
+function cleanStringOfNewLinesAndTabs(str: string): string {
+ if (!str) {
+ return '';
+ }
+ let trimmedSQL = str.replace(/\n/g, ' ');
+ trimmedSQL = trimmedSQL.replaceAll(/ {2,50}/g, ' ');
+ trimmedSQL = trimmedSQL.replace(/\t+/g, '');
+ return trimmedSQL.trim();
+}
+
+function convertLogicToSQL(logic: Logic, nodeAlias: string): string {
+ if (!logic || !Array.isArray(logic)) return '';
+
+ const operator = logic[0];
+ const left = logic[1];
+ const right = logic[2];
+
+ // Handle field references (e.g., @p1.name)
+ if (typeof left === 'string' && left.startsWith('@')) {
+ const [alias, field] = left.substring(1).split('.');
+ if (alias === nodeAlias) {
+ return `${alias}.${field} ${operator} '${right.replace(/^"(.*)"$/, '$1')}'`;
+ }
+ }
+ return '';
+}
+
+function buildNodeCTE(node: NodeStruct | undefined, logic?: Logic, parent: NodeStruct | undefined = undefined): string[] {
+ if (!node || !node.label || !node.id) {
+ return [];
+ }
+ const nodeParts: string[] = [];
+ const logicClause = convertLogicToSQL(logic, node.id);
+ const whereClause = logic && logicClause ? `WHERE ${convertLogicToSQL(logic, node.id)}` : '';
+ const mainNode = `
+ SELECT jsonb_build_object(
+ '_id', concat('${node.label}-', ${node.id}.id::text),
+ 'label', '${node.label}',
+ 'attributes', to_jsonb(${node.id}) - 'id'
+ ) AS node
+ FROM ${node.label} ${node.id}
+ ${
+ parent && parent.label && parent.id && parent.relation && parent.relation.label
+ ? `INNER JOIN ${parent.label} ${parent.id} ON ${parent.id}.${parent.relation.label.split('.')[0]} = ${node.id}.${
+ parent.relation.label.split('.')[1]
+ }`
+ : ''
+ }
+ ${whereClause}`;
+ nodeParts.push(cleanStringOfNewLinesAndTabs(mainNode));
+ buildNodeCTE(node.relation?.node, logic, node).forEach(part => nodeParts.push(part));
+ return nodeParts;
+}
+
+function buildEdgesCTE(node: NodeStruct | undefined, logic?: Logic, parent: NodeStruct | undefined = undefined): string[] {
+ if (!node || !node.relation || !node.relation.node || !node.id || !node.relation.node.id || !node.relation.label) {
+ return [];
+ }
+ const logicClause = convertLogicToSQL(logic, node.id);
+ const whereClause = logic && logicClause ? `WHERE ${convertLogicToSQL(logic, node.id)}` : '';
+
+ const edge = `
+ SELECT jsonb_agg(
+ jsonb_build_object(
+ '_id', concat('${node.label}-', ${node.id}.id::text, '-${node.relation.node.label}-', ${node.relation.node.id}.id::text),
+ 'label', '${node.relation.label}',
+ 'attributes', '{}'::jsonb,
+ 'from', ${node.id}.id::text,
+ 'to', ${node.relation.node.id}.id::text
+ )
+ ) AS edge
+ FROM ${node.label} ${node.id}
+ INNER JOIN ${node.relation.node.label} ${node.relation.node.id} ON ${node.id}.${node.relation.label.split('.')[0]} = ${
+ node.relation.node.id
+ }.${node.relation.label.split('.')[1]}
+ ${whereClause}`;
+ const ret = cleanStringOfNewLinesAndTabs(edge);
+ return [ret, ...buildEdgesCTE(node.relation.node, logic, node)];
+}
+
+function buildNodeCountsCTE(node: NodeStruct | undefined): string[] {
+ if (!node || !node.label || !node.id) {
+ return [];
+ }
+ const nodeParts: string[] = [];
+ const mainNode = `SELECT '${node.label}' AS label FROM ${node.label}`;
+ nodeParts.push(cleanStringOfNewLinesAndTabs(mainNode));
+ buildNodeCountsCTE(node.relation?.node).forEach(part => nodeParts.push(part));
+ return nodeParts;
+}
+
+function buildEdgeCountsCTE(node: NodeStruct | undefined): string[] {
+ if (!node || !node.relation || !node.relation.node || !node.id || !node.relation.node.id || !node.relation.label) {
+ return [];
+ }
+
+ const edge = `
+ SELECT '${node.relation.label}' AS label, COUNT(*) AS count
+ FROM ${node.label} ${node.id}
+ JOIN ${node.relation.node.label} ${node.relation.node.id}
+ ON ${node.id}.${node.relation.label.split('.')[0]} = ${node.relation.node.id}.${node.relation.label.split('.')[1]}
+ GROUP BY label`;
+ const ret = cleanStringOfNewLinesAndTabs(edge);
+ return [ret, ...buildEdgeCountsCTE(node.relation.node)];
+}
+
+export function query2SQL(query: BackendQueryFormat): QueryText {
+ const nodesCTE = [...new Set(query.query.flatMap(path => buildNodeCTE(path.node, query.logic)))];
+ const edgesCTE = [...new Set(query.query.flatMap(path => buildEdgesCTE(path.node)))];
+ const nodeCountsCTE = [...new Set(query.query.flatMap(path => buildNodeCountsCTE(path.node)))];
+ const edgeCountsCTE = [...new Set(query.query.flatMap(path => buildEdgeCountsCTE(path.node)))];
+
+ const mainQuery = `WITH raw_nodes AS (
+ SELECT node
+ FROM (
+ ${nodesCTE.join(' UNION ALL ')}
+ ) sub
+ ${query.limit ? `LIMIT ${query.limit}` : ''}
+ ),
+ nodes_cte AS (
+ SELECT jsonb_agg(node) AS nodes
+ FROM raw_nodes
+ ),
+ edges_cte AS (
+ SELECT jsonb_agg(edge) AS edges
+ FROM (
+ ${edgesCTE.join(' UNION ALL ')}
+ ) aggregated_edges
+ ),
+ node_counts AS (
+ SELECT label, COUNT(*) AS count
+ FROM (
+ ${nodeCountsCTE.join(' UNION ALL ')}
+ ) t
+ GROUP BY label
+ ),
+ edge_counts AS (
+ ${edgeCountsCTE.join(' UNION ALL ')}
+ )
+ SELECT jsonb_build_object(
+ 'nodes', nodes_cte.nodes,
+ 'edges', COALESCE(edges_cte.edges, '[]'::jsonb),
+ 'nodeCounts', COALESCE( (SELECT jsonb_agg(jsonb_build_object('label', label, 'count', count))
+ FROM node_counts), '[]'::jsonb ),
+ 'edgeCounts', COALESCE( (SELECT jsonb_agg(jsonb_build_object('label', label, 'count', count))
+ FROM edge_counts), '[]'::jsonb )
+ ) AS result FROM nodes_cte, edges_cte;`;
+
+ return {
+ query: mainQuery,
+ countQuery: '',
+ };
+}
diff --git a/src/queryExecution/sql/queryConverterSql.test.ts b/src/queryExecution/sql/queryConverterSql.test.ts
index 3773a70..7de4d9f 100644
--- a/src/queryExecution/sql/queryConverterSql.test.ts
+++ b/src/queryExecution/sql/queryConverterSql.test.ts
@@ -166,7 +166,7 @@ FROM nodes_cte, edges_cte;`;
const query: BackendQueryFormat = {
saveStateID: 'test',
return: ['*'],
- logic: ['==', '@uu.username', '"Username1"'],
+ logic: ['!=', '@uu.username', '"Username1"'],
query: [
{
id: 'path1',
diff --git a/src/queryExecution/sql/queryConverterSql.ts b/src/queryExecution/sql/queryConverterSql.ts
index c370a1e..6d91a1d 100644
--- a/src/queryExecution/sql/queryConverterSql.ts
+++ b/src/queryExecution/sql/queryConverterSql.ts
@@ -1,7 +1,7 @@
-import type { BackendQueryFormat, NodeStruct } from 'ts-common';
+import type { AllLogicStatement, AnyStatement, BackendQueryFormat, NodeStruct, QueryStruct } from 'ts-common';
import type { QueryText } from '../model';
-
-type Logic = any;
+import { extractLogicSQL } from './convertLogicToSQL';
+import type { QueryCacheData } from '../cypher/converter/model';
function cleanStringOfNewLinesAndTabs(str: string): string {
if (!str) {
@@ -13,146 +13,272 @@ function cleanStringOfNewLinesAndTabs(str: string): string {
return trimmedSQL.trim();
}
-function convertLogicToSQL(logic: Logic, nodeAlias: string): string {
- if (!logic || !Array.isArray(logic)) return '';
+// function buildNodeCTE(path: QueryStruct, logic: AnyStatement | undefined): { node: string; edges: string; rows: string[] } {
+// const node = path.node;
+// const pathId = path.id;
+
+// if (!node || !node.label || !node.id) {
+// return { node: '', edges: '', rows: [] };
+// }
+
+// const loopThoughNodes = (node: NodeStruct): NodeStruct[] => {
+// if (node.relation && node.relation.node) {
+// return [node, ...loopThoughNodes(node.relation.node)];
+// }
+// return [node];
+// };
+// const allNodes = loopThoughNodes(node);
+
+// const where = logic ? extractLogicSQL(logic, { nodes: allNodes }).logic : undefined;
+// const rowList = allNodes.map(n => `${n.id}_row`);
+// const select = `
+// SELECT ${[
+// ...allNodes.map(n => `to_jsonb(${n.id}) AS ${n.id}_row`),
+// ...allNodes
+// .filter(n => n.relation && n!.relation!.node)
+// .map(
+// n =>
+// `${n.id}.${n!.relation!.label!.split('.')[0]}::text AS ${n.id}_${n!.relation!.label!.split('.')[0]}, ${n!.relation!.node!.id}.${
+// n!.relation!.label!.split('.')[1]
+// }::text AS ${n!.relation!.node!.id}_${n!.relation!.label!.split('.')[1]}`,
+// ),
+// ].join(', ')}
+// FROM ${node.label} ${node.id}
+// ${allNodes
+// .filter(n => n?.relation?.node?.label && n?.relation?.node?.id)
+// .map(
+// n =>
+// `INNER JOIN ${n!.relation!.node!.label} ${n!.relation!.node!.id} ON ${n.id}.${n!.relation!.label!.split('.')[0]}::text = ${
+// n!.relation!.node!.id
+// }.${n!.relation!.label!.split('.')[1]}::text`,
+// )
+// .join('\n')}
+// ${where ? `WHERE ${where}` : ''}`;
+
+// const edges = buildEdgesCTE(node, rowList, pathId).join(' UNION ALL ');
+
+// return { node: select, edges, rows: rowList };
+// }
+
+// function buildEdgesCTE(node: NodeStruct | undefined, rowList: string[], pathId: string): string[] {
+// if (!node || !node.relation || !node.relation.node || !node.id || !node.relation.node.id || !node.relation.label) {
+// return [];
+// }
+
+// const edge = `
+// SELECT jsonb_build_object(
+// '_id', concat('${node.label}-', ${node.id}_${node.relation.label.split('.')[0]}::text, '-${node.relation.node.label}-', ${
+// node.relation.node.id
+// }_${node.relation.label.split('.')[1]}::text),
+// 'label', '${node.relation.label}',
+// 'attributes', '{}'::jsonb,
+// 'from', ${node.id}_${node.relation.label.split('.')[0]}::text,
+// 'to', ${node.relation.node.id}_${node.relation.label.split('.')[1]}::text
+// ) AS edge,
+// ${rowList.join(',\n')}
+// FROM ${pathId}`;
+
+// const ret = cleanStringOfNewLinesAndTabs(edge);
+// return [ret, ...buildEdgesCTE(node.relation.node, rowList, pathId)];
+// }
+
+// function buildNodeCountsCTE(node: NodeStruct | undefined): string[] {
+// if (!node || !node.label || !node.id) {
+// return [];
+// }
+// const nodeParts: string[] = [];
+// const mainNode = `SELECT '${node.label}' AS label FROM ${node.label}`;
+// nodeParts.push(cleanStringOfNewLinesAndTabs(mainNode));
+// buildNodeCountsCTE(node.relation?.node).forEach(part => nodeParts.push(part));
+// return nodeParts;
+// }
- const operator = logic[0];
- const left = logic[1];
- const right = logic[2];
+// function buildEdgeCountsCTE(node: NodeStruct | undefined): string[] {
+// if (!node || !node.relation || !node.relation.node || !node.id || !node.relation.node.id || !node.relation.label) {
+// return [];
+// }
- // Handle field references (e.g., @p1.name)
- if (typeof left === 'string' && left.startsWith('@')) {
- const [alias, field] = left.substring(1).split('.');
- if (alias === nodeAlias) {
- return `${alias}.${field} ${operator} '${right.replace(/^"(.*)"$/, '$1')}'`;
+// const edge = `
+// SELECT '${node.relation.label}' AS label, COUNT(*) AS count
+// FROM ${node.label} ${node.id}
+// JOIN ${node.relation.node.label} ${node.relation.node.id}
+// ON ${node.id}.${node.relation.label.split('.')[0]} = ${node.relation.node.id}.${node.relation.label.split('.')[1]}
+// GROUP BY label`;
+// const ret = cleanStringOfNewLinesAndTabs(edge);
+// return [ret, ...buildEdgeCountsCTE(node.relation.node)];
+// }
+
+export function query2SQL(query: BackendQueryFormat): QueryText {
+ const loopThoughNodes = (node: NodeStruct): NodeStruct[] => {
+ if (node.relation && node.relation.node) {
+ return [node, ...loopThoughNodes(node.relation.node)];
}
- }
- return '';
-}
+ return [node];
+ };
+ const allNodes = query.query.flatMap(path => loopThoughNodes(path.node));
+ const allUniqueNodes: Record<
+ string,
+ { label: string; joins: { from: string; leftJoin: string; rightJoin: string; to: string; fromLabel: string; toLabel: string }[] }
+ > = Object.fromEntries(
+ allNodes
+ .map(n => [n.id, { label: n.label, joins: [] }])
+ .filter((node, index, self) => {
+ return index === self.findIndex(t => t[0] === node[0] && t[1] === node[1]);
+ }),
+ );
+ allNodes.forEach(n => {
+ if (n.id && n.relation && n.relation.node && n.relation.node.id && n.label && n.relation.node.label) {
+ allUniqueNodes[n.id].joins.push({
+ from: n.id,
+ leftJoin: n!.relation!.label!.split('.')[0],
+ rightJoin: n!.relation!.label!.split('.')[1],
+ to: n!.relation!.node!.id,
+ fromLabel: n!.label,
+ toLabel: n!.relation!.node!.label,
+ });
+ }
+ });
-function buildNodeCTE(node: NodeStruct | undefined, logic?: Logic, parent: NodeStruct | undefined = undefined): string[] {
- if (!node || !node.label || !node.id) {
- return [];
- }
- const nodeParts: string[] = [];
- const logicClause = convertLogicToSQL(logic, node.id);
- const whereClause = logic && logicClause ? `WHERE ${convertLogicToSQL(logic, node.id)}` : '';
- const mainNode = `
- SELECT jsonb_build_object(
- '_id', concat('${node.label}-', ${node.id}.id::text),
- 'label', '${node.label}',
- 'attributes', to_jsonb(${node.id}) - 'id'
- ) AS node
- FROM ${node.label} ${node.id}
- ${
- parent && parent.label && parent.id && parent.relation && parent.relation.label
- ? `INNER JOIN ${parent.label} ${parent.id} ON ${parent.id}.${parent.relation.label.split('.')[0]} = ${node.id}.${
- parent.relation.label.split('.')[1]
- }`
- : ''
- }
- ${whereClause}`;
- nodeParts.push(cleanStringOfNewLinesAndTabs(mainNode));
- buildNodeCTE(node.relation?.node, logic, node).forEach(part => nodeParts.push(part));
- return nodeParts;
-}
+ console.log(JSON.stringify(allNodes, null, 2));
-function buildEdgesCTE(node: NodeStruct | undefined, logic?: Logic, parent: NodeStruct | undefined = undefined): string[] {
- if (!node || !node.relation || !node.relation.node || !node.id || !node.relation.node.id || !node.relation.label) {
- return [];
- }
- const logicClause = convertLogicToSQL(logic, node.id);
- const whereClause = logic && logicClause ? `WHERE ${convertLogicToSQL(logic, node.id)}` : '';
-
- const edge = `
- SELECT jsonb_agg(
- jsonb_build_object(
- '_id', concat('${node.label}-', ${node.id}.id::text, '-${node.relation.node.label}-', ${node.relation.node.id}.id::text),
- 'label', '${node.relation.label}',
- 'attributes', '{}'::jsonb,
- 'from', ${node.id}.id::text,
- 'to', ${node.relation.node.id}.id::text
+ const nodeEntries = Object.entries(allUniqueNodes);
+ const where = query.logic ? extractLogicSQL(query.logic, { nodes: allNodes }).logic : undefined;
+
+ const firstNode = nodeEntries[0][1];
+
+ const relIDSql = Object.values(allUniqueNodes)
+ .map(n =>
+ n.joins.map(j => `${j.from}.${j.leftJoin} AS ${j.from}_${j.leftJoin}, ${j.to}.${j.rightJoin} AS ${j.to}_${j.rightJoin}`).join(', '),
)
- ) AS edge
- FROM ${node.label} ${node.id}
- INNER JOIN ${node.relation.node.label} ${node.relation.node.id} ON ${node.id}.${node.relation.label.split('.')[0]} = ${
- node.relation.node.id
- }.${node.relation.label.split('.')[1]}
- ${whereClause}`;
- const ret = cleanStringOfNewLinesAndTabs(edge);
- return [ret, ...buildEdgesCTE(node.relation.node, logic, node)];
-}
+ .filter(s => s.length > 0);
+ const select = [...nodeEntries.map(n => `to_jsonb(${n[0]}) AS ${n[0]}_row`), ...relIDSql].join(', ');
-function buildNodeCountsCTE(node: NodeStruct | undefined): string[] {
- if (!node || !node.label || !node.id) {
- return [];
- }
- const nodeParts: string[] = [];
- const mainNode = `SELECT '${node.label}' AS label FROM ${node.label}`;
- nodeParts.push(cleanStringOfNewLinesAndTabs(mainNode));
- buildNodeCountsCTE(node.relation?.node).forEach(part => nodeParts.push(part));
- return nodeParts;
-}
+ let sql = `SELECT ${select} FROM ${firstNode.label} ${nodeEntries[0][0]}`;
+ // let sql = `SELECT jsonb_agg(row_to_json(t)) FROM ${firstNode.label} ${nodeEntries[0][0]}`;
-function buildEdgeCountsCTE(node: NodeStruct | undefined): string[] {
- if (!node || !node.relation || !node.relation.node || !node.id || !node.relation.node.id || !node.relation.label) {
- return [];
+ let joins = '';
+ for (let i = 0; i < nodeEntries.length; i++) {
+ const [nodeId, node] = nodeEntries[i];
+ if (node.joins) {
+ node.joins.forEach(j => {
+ joins += ` INNER JOIN ${allUniqueNodes[j.to].label} ${j.to} ON ${j.from}.${j.leftJoin}::text = ${j.to}.${j.rightJoin}::text`;
+ });
+ }
}
- const edge = `
- SELECT '${node.relation.label}' AS label, COUNT(*) AS count
- FROM ${node.label} ${node.id}
- JOIN ${node.relation.node.label} ${node.relation.node.id}
- ON ${node.id}.${node.relation.label.split('.')[0]} = ${node.relation.node.id}.${node.relation.label.split('.')[1]}
- GROUP BY label`;
- const ret = cleanStringOfNewLinesAndTabs(edge);
- return [ret, ...buildEdgeCountsCTE(node.relation.node)];
-}
+ sql += joins;
+ sql += where ? ` WHERE ${where}` : '';
+ const sqlNoLimit = sql;
+ sql += query.limit ? ` LIMIT ${query.limit}` : '';
-export function query2SQL(query: BackendQueryFormat): QueryText {
- const nodesCTE = [...new Set(query.query.flatMap(path => buildNodeCTE(path.node, query.logic)))];
- const edgesCTE = [...new Set(query.query.flatMap(path => buildEdgesCTE(path.node)))];
- const nodeCountsCTE = [...new Set(query.query.flatMap(path => buildNodeCountsCTE(path.node)))];
- const edgeCountsCTE = [...new Set(query.query.flatMap(path => buildEdgeCountsCTE(path.node)))];
+ const nodes_cte = `SELECT jsonb_agg(row_to_json(t)) AS nodes FROM (${nodeEntries
+ .map(n => `SELECT DISTINCT ${n[0]}_row AS node FROM combined`)
+ .join(' UNION ALL ')}) AS t`;
- const mainQuery = `WITH raw_nodes AS (
- SELECT node
- FROM (
- ${nodesCTE.join(' UNION ALL ')}
- ) sub
- ${query.limit ? `LIMIT ${query.limit}` : ''}
- ),
+ const allUniqueEdges = Object.values(allUniqueNodes)
+ .map(n =>
+ n.joins
+ .map(j => {
+ return `SELECT jsonb_build_object(
+ '_id', concat('${n.label}-', md5(${j.from}_row::text), '-${allUniqueNodes[j.to].label}-', md5(${j.to}_row::text)),
+ 'label', '${j.leftJoin}.${j.rightJoin}',
+ 'attributes', '{}'::jsonb,
+ 'from', md5(${j.from}_row::text),
+ 'to', md5(${j.to}_row::text)
+ ) AS edge
+ FROM combined`;
+ })
+ .filter(s => s.length > 0)
+ .join(' UNION ALL '),
+ )
+ .filter(s => s.length > 0)
+ .join(' UNION ALL ');
+
+ // console.log(JSON.stringify(allUniqueNodes, null, 2));
+ // const rowList = allNodes.map(n => `${n.id}_row`);
+ // const select = `
+ // SELECT ${allNodes.map(n => `to_jsonb(${n.id}) AS ${n.id}_row`).join(', ')},
+ // ${allNodes
+ // .filter(n => n.relation && n!.relation!.node)
+ // .map(
+ // n =>
+ // `${n.id}.${n!.relation!.label!.split('.')[0]}::text AS ${n.id}_${n!.relation!.label!.split('.')[0]}, ${n!.relation!.node!.id}.${
+ // n!.relation!.label!.split('.')[1]
+ // }::text AS ${n!.relation!.node!.id}_${n!.relation!.label!.split('.')[1]}`,
+ // )
+ // .join(', ')}
+ // FROM ${node.label} ${node.id}
+ // ${allNodes
+ // .filter(n => n?.relation?.node?.label && n?.relation?.node?.id)
+ // .map(
+ // n =>
+ // `INNER JOIN ${n!.relation!.node!.label} ${n!.relation!.node!.id} ON ${n.id}.${n!.relation!.label!.split('.')[0]}::text = ${
+ // n!.relation!.node!.id
+ // }.${n!.relation!.label!.split('.')[1]}::text`,
+ // )
+ // .join('\n')}
+ // ${where ? `WHERE ${where}` : ''}`;
+
+ // const edges = buildEdgesCTE(node, rowList, pathId).join(' UNION ALL ');
+
+ // const nodesCTE = query.query.flatMap(path => ({ ...buildNodeCTE(path, query.logic), path: path }));
+ // const nodeCountsCTE = [...new Set(query.query.flatMap(path => buildNodeCountsCTE(path.node)))];
+ // const edgeCountsCTE = [...new Set(query.query.flatMap(path => buildEdgeCountsCTE(path.node)))];
+
+ const mainQuery = `WITH combined AS (${sql}),
nodes_cte AS (
- SELECT jsonb_agg(node) AS nodes
- FROM raw_nodes
+ SELECT jsonb_agg(row_to_json(t)) AS nodes FROM (${nodeEntries
+ .map(
+ ([nodeId, node]) => `
+ SELECT
+ md5(${nodeId}_row::text) AS _id,
+ '${node.label}' AS label,
+ ${nodeId}_row - 'id' AS attributes
+ FROM combined`,
+ )
+ .join(' UNION ALL ')}) AS t
),
edges_cte AS (
- SELECT jsonb_agg(edge) AS edges
- FROM (
- ${edgesCTE.join(' UNION ALL ')}
- ) aggregated_edges
+ SELECT jsonb_agg(edge) AS edges
+ FROM (
+ ${allUniqueEdges.length > 0 ? allUniqueEdges : 'SELECT null AS edge WHERE FALSE'}
+ ) AS e
),
node_counts AS (
- SELECT label, COUNT(*) AS count
+ SELECT jsonb_agg(jsonb_build_object('label', label, 'count', count)) AS nodeCounts
FROM (
- ${nodeCountsCTE.join(' UNION ALL ')}
- ) t
- GROUP BY label
- ),
- edge_counts AS (
- ${edgeCountsCTE.join(' UNION ALL ')}
+ SELECT label, COUNT(*) AS count
+ FROM (
+ ${[...new Set(nodeEntries.map(([_, node]) => node.label))]
+ .map(label => `SELECT '${label}' AS label FROM ${label}`)
+ .join(' UNION ALL ')}
+ ) AS node_labels
+ GROUP BY label
+ ) AS counts
)
SELECT jsonb_build_object(
'nodes', nodes_cte.nodes,
'edges', COALESCE(edges_cte.edges, '[]'::jsonb),
- 'nodeCounts', COALESCE( (SELECT jsonb_agg(jsonb_build_object('label', label, 'count', count))
- FROM node_counts), '[]'::jsonb ),
- 'edgeCounts', COALESCE( (SELECT jsonb_agg(jsonb_build_object('label', label, 'count', count))
- FROM edge_counts), '[]'::jsonb )
+ 'nodeCounts', COALESCE((SELECT nodeCounts FROM node_counts), '[]'::jsonb)
) AS result FROM nodes_cte, edges_cte;`;
+ // 'nodeCounts', COALESCE( (SELECT jsonb_agg(jsonb_build_object('label', label, 'count', count))
+ // FROM node_counts), '[]'::jsonb ),
+ // 'edgeCounts', COALESCE( (SELECT jsonb_agg(jsonb_build_object('label', label, 'count', count))
+ // FROM edge_counts), '[]'::jsonb )
+ // node_counts AS (
+ // SELECT label, COUNT(*) AS count
+ // FROM (
+ // ${nodesCTE.map(node => node.node).join(' UNION ALL ')}
+ // ) t
+ // GROUP BY label
+ // ),
+ // edge_counts AS (
+ // ${nodesCTE.map(node => node.edges).join(' UNION ALL ')}
+ // )
+
return {
- query: mainQuery,
+ query: cleanStringOfNewLinesAndTabs(mainQuery),
countQuery: '',
};
}
diff --git a/src/queryExecution/sql/queryResultParser.ts b/src/queryExecution/sql/queryResultParser.ts
deleted file mode 100644
index a3dc7f6..0000000
--- a/src/queryExecution/sql/queryResultParser.ts
+++ /dev/null
@@ -1,121 +0,0 @@
-import { log } from '../../logger';
-import type { CountQueryResultFromBackend, EdgeQueryResult, NodeQueryResult } from 'ts-common';
-import type { GraphQueryResultFromBackend } from 'ts-common';
-import { type QueryResult } from 'pg';
-
-// Adjusted to handle a Postgres QueryResult object.
-export function parseSQLQueryResult(result: QueryResult, returnType: 'nodelink' | 'table' = 'nodelink'): GraphQueryResultFromBackend {
- // ...existing error handling code...
- try {
- // Extract rows if present.
- switch (returnType) {
- case 'nodelink':
- return parseSQLNodeLinkQuery(result.rows);
- case 'table':
- log.error(`Table format not supported yet`);
- throw new Error('Table format not supported yet');
- default:
- log.error(`Error Unknown query Format`);
- throw new Error('Unknown query Format');
- }
- } catch (err) {
- log.error(`Error executing query`, err);
- throw err;
- }
-}
-
-// Adjusted to handle a Postgres QueryResult object.
-export function parseCountSQLQueryResult(result: QueryResult): CountQueryResultFromBackend {
- try {
- const countResult: CountQueryResultFromBackend = { updatedAt: Date.now() };
- const rows = result.rows as Record<string, any>[];
- if (rows.length > 0) {
- const row = rows[0];
- for (const key in row) {
- if (Object.prototype.hasOwnProperty.call(row, key)) {
- countResult[key] = Number(row[key]);
- }
- }
- }
- return countResult;
- } catch (err) {
- log.error(`Error executing query`, err);
- throw err;
- }
-}
-
-// Helper function to build a graph result (nodelink) from SQL rows.
-function parseSQLNodeLinkQuery(rows: Record<string, any>[]): GraphQueryResultFromBackend {
- const nodes: NodeQueryResult[] = [];
- const edges: EdgeQueryResult[] = [];
- const seenNodes = new Set<string>();
- const seenEdges = new Set<string>();
-
- for (const row of rows) {
- // If the row doesn't carry a "type", assume it represents a node.
- if (!row.type) {
- if (!seenNodes.has(row.id)) {
- nodes.push({
- _id: row.id,
- label: row.name, // using "name" as the label
- attributes: { ...row },
- });
- seenNodes.add(row.id);
- }
- } else if (row.type === 'node') {
- if (!seenNodes.has(row.id)) {
- nodes.push({
- _id: row.id,
- label: row.label,
- attributes: row.attributes, // Assumed to be a plain object.
- });
- seenNodes.add(row.id);
- }
- } else if (row.type === 'edge') {
- if (!seenEdges.has(row.id)) {
- edges.push({
- _id: row.id,
- label: row.label,
- from: row.from,
- to: row.to,
- attributes: { ...row.attributes, type: row.label },
- });
- seenEdges.add(row.id);
- }
- } else if (row.type === 'path') {
- // If a path, assume arrays "nodes" and "edges" exist.
- if (Array.isArray(row.nodes)) {
- for (const node of row.nodes) {
- if (!seenNodes.has(node.id)) {
- nodes.push({
- _id: node.id,
- label: node.label,
- attributes: node.attributes,
- });
- seenNodes.add(node.id);
- }
- }
- }
- if (Array.isArray(row.edges)) {
- for (const edge of row.edges) {
- if (!seenEdges.has(edge.id)) {
- edges.push({
- _id: edge.id,
- label: edge.label,
- from: edge.from,
- to: edge.to,
- attributes: { ...edge.attributes, type: edge.label },
- });
- seenEdges.add(edge.id);
- }
- }
- }
- } else {
- log.warn(`Ignoring unknown row type: ${row.type}`);
- }
- }
-
- return { nodes, edges };
-}
-
-// Removed neo4j-specific helper functions.
diff --git a/src/readers/services/sqlService.ts b/src/readers/services/sqlService.ts
index fed1c52..d6451e5 100644
--- a/src/readers/services/sqlService.ts
+++ b/src/readers/services/sqlService.ts
@@ -2,10 +2,7 @@ import { type DbConnection, type GraphQueryResultMetaFromBackend, graphQueryBack
import { PgConnection } from 'ts-common/databaseConnection/postgres';
import { log } from '../../logger';
import { QUERY_CACHE_DURATION, redis } from '../../variables';
-import { parseCountCypherQueryResult, parseCypherQueryResult } from '../../queryExecution/cypher/queryResultParser';
-import { cacheCheck } from './cache';
import type { QueryText } from '../../queryExecution/model';
-import { parseCountSQLQueryResult, parseSQLQueryResult } from '../../queryExecution/sql/queryResultParser';
export const sqlQueryService = async (
db: DbConnection,
@@ -30,16 +27,14 @@ export const sqlQueryService = async (
const connection = new PgConnection(db);
try {
- const [result, countResult] = await connection.run([queryText.query, queryText.countQuery]);
- // console.log('result:', result);
+ const [result] = await connection.run([queryText.query]);
+ // console.log('result:', result.rows[0].result);
// console.log('countResult:', countResult);
-
- const graph = parseSQLQueryResult(result);
- log.info('Parsed graph:', result);
- const countGraph = parseCountSQLQueryResult(countResult);
+ const queryResult = result.rows[0].result as GraphQueryResultMetaFromBackend;
+ queryResult.nodes = queryResult.nodes ?? [];
// calculate metadata
- const graphQueryResult = graphQueryBackend2graphQuery(graph, countGraph);
+ const graphQueryResult = graphQueryBackend2graphQuery(queryResult, queryResult.nodeCounts);
graphQueryResult.nodeCounts.updatedAt = Date.now();
// cache result
@@ -55,8 +50,8 @@ export const sqlQueryService = async (
return graphQueryResult;
} catch (error) {
- log.error('Error parsing query result:', queryText, error);
- throw new Error('Error parsing query result');
+ log.error('Error parsing sql query result:', queryText, error);
+ throw new Error('Error parsing sql query result');
} finally {
connection.close();
}
--
GitLab