Add GROUP_CONCAT support for ORDER BY and SEPARATOR clauses

.github/copilot-instructions.md

@@ -7,7 +7,7 @@ AlaSQL is an open source SQL database for JavaScript with a focus on query speed
 ## When Implementing Features
 
 1. **Understand the issue thoroughly** - Read related test cases and existing code
-2. **Write a test first** - Copy test/test000.js into a new file called `test/test###.js` where where `###` is the id of the issue we are trying to solve
+2. **Write a test first** - Copy test/test000.js into a new file called `test/test###.js` where where `###` is the id of the issue we are trying to solve.
 3. **Verify test fails** - Run `yarn test` to confirm the test catches the issue
 4. **Implement the fix** - Modify appropriate file(s) in `src/`
   - If you modify the grammar in `src/alasqlgrammar.jison`, run `yarn jison && yarn test` to regenerate the parser and verify
@@ -37,3 +37,12 @@ yarn format
 - `src/alasqlparser.js` - Generated from Jison grammar (modify the `.jison` file instead)
 - `.min.js` files - Generated during build
 
+
+## Plesae note 
+
+- Alasql is meant to return `undefined` instead of `null` (unline regular SQL engines)
+
+## Resources
+
+- [AlaSQL Documentation](https://github.com/alasql/alasql/wiki)
+- [Issue Tracker](https://github.com/AlaSQL/alasql/issues)

src/423groupby.js

@@ -6,6 +6,24 @@
 //
 */
 
+/**
+ * Helper to get GROUP_CONCAT parameters for compilation
+ * @param {Object} col - Column with aggregatorid and funcid
+ * @returns {string} - Comma-separated parameter string for aggregator call
+ */
+function getGroupConcatParams(col) {
+	if (!col.funcid || col.funcid.toUpperCase() !== 'GROUP_CONCAT') {
+		return '';
+	}
+	const separator = col.separator !== undefined ? JSON.stringify(col.separator) : 'undefined';
+	// Extract direction from order expressions (MySQL GROUP_CONCAT orders by the value itself)
+	const orderDir =
+		col.order && col.order.length > 0 && col.order[0].direction
+			? JSON.stringify(col.order[0].direction)
+			: 'undefined';
+	return `,${separator},${orderDir}`;
+}
+
 /**
  Compile group of statements
  */
@@ -117,15 +135,15 @@ yy.Select.prototype.compileGroup = function (query) {
 						if ('funcid' in col.expression) {
 							let colexp1 = colExpIfFunIdExists(col.expression);
 
-							return `'${colas}': (__alasql_tmp = ${colexp}, typeof __alasql_tmp == 'number' || typeof __alasql_tmp == 'bigint' || (typeof __alasql_tmp == 'object' && (typeof Number(__alasql_tmp) == 'number' || __alasql_tmp instanceof Date)) ? __alasql_tmp : undefined),`;
+							return `'${colas}': (__alasql_tmp = ${colexp}, __alasql_tmp !== null && (typeof __alasql_tmp == 'number' || typeof __alasql_tmp == 'bigint' || (typeof __alasql_tmp == 'object' && (typeof Number(__alasql_tmp) == 'number' || __alasql_tmp instanceof Date))) ? __alasql_tmp : undefined),`;
 						}
-						return `'${colas}': (__alasql_tmp = ${colexp}, typeof __alasql_tmp == 'number' || typeof __alasql_tmp == 'bigint' || (typeof __alasql_tmp == 'object' && (typeof Number(__alasql_tmp) == 'number' || __alasql_tmp instanceof Date)) ? __alasql_tmp : undefined),`;
+						return `'${colas}': (__alasql_tmp = ${colexp}, __alasql_tmp !== null && (typeof __alasql_tmp == 'number' || typeof __alasql_tmp == 'bigint' || (typeof __alasql_tmp == 'object' && (typeof Number(__alasql_tmp) == 'number' || __alasql_tmp instanceof Date))) ? __alasql_tmp : undefined),`;
 					} else if (col.aggregatorid === 'MAX') {
 						if ('funcid' in col.expression) {
 							let colexp1 = colExpIfFunIdExists(col.expression);
-							return `'${colas}': (__alasql_tmp = ${colexp}, typeof __alasql_tmp == 'number' || typeof __alasql_tmp == 'bigint' || (typeof __alasql_tmp == 'object' && (typeof Number(__alasql_tmp) == 'number' || __alasql_tmp instanceof Date)) ? __alasql_tmp : undefined),`;
+							return `'${colas}': (__alasql_tmp = ${colexp}, __alasql_tmp !== null && (typeof __alasql_tmp == 'number' || typeof __alasql_tmp == 'bigint' || (typeof __alasql_tmp == 'object' && (typeof Number(__alasql_tmp) == 'number' || __alasql_tmp instanceof Date))) ? __alasql_tmp : undefined),`;
 						}
-						return `'${colas}' : (typeof ${colexp} == 'number' || typeof ${colexp} == 'bigint' ? ${colexp} : typeof ${colexp} == 'object' ?
+						return `'${colas}' : (${colexp} !== null && (typeof ${colexp} == 'number' || typeof ${colexp} == 'bigint') ? ${colexp} : ${colexp} !== null && typeof ${colexp} == 'object' ?
 							typeof Number(${colexp}) == 'number' ? ${colexp} : undefined : undefined),`;
 					} else if (col.aggregatorid === 'ARRAY') {
 						return `'${colas}':[${colexp}],`;
@@ -145,7 +163,8 @@ yy.Select.prototype.compileGroup = function (query) {
 						return '';
 					} else if (col.aggregatorid === 'REDUCE') {
 						query.aggrKeys.push(col);
-						return `'${colas}':alasql.aggr['${col.funcid}'](${colexp},undefined,1),`;
+						const extraParams = getGroupConcatParams(col);
+						return `'${colas}':alasql.aggr['${col.funcid}'](${colexp},undefined,1${extraParams}),`;
 					}
 					return '';
 				}
@@ -415,8 +434,9 @@ yy.Select.prototype.compileGroup = function (query) {
 							g['${colas}']=${col.expression.toJS('g', -1)};
 							${post}`;
 					} else if (col.aggregatorid === 'REDUCE') {
+						const extraParams = getGroupConcatParams(col);
 						return `${pre}
-							g['${colas}'] = alasql.aggr.${col.funcid}(${colexp},g['${colas}'],2);
+							g['${colas}'] = alasql.aggr.${col.funcid}(${colexp},g['${colas}'],2${extraParams});
 							${post}`;
 					}
 

src/55functions.js

@@ -281,26 +281,76 @@ stdfn.CONCAT_WS = function () {
 // TRIM
 
 // Aggregator for joining strings
-alasql.aggr.group_concat = alasql.aggr.GROUP_CONCAT = function (v, s, stage) {
+// Can be called with options: alasql.aggr.GROUP_CONCAT(v, s, stage, separator, orderDirection)
+alasql.aggr.group_concat = alasql.aggr.GROUP_CONCAT = function (
+	v,
+	s,
+	stage,
+	separator,
+	orderDirection
+) {
+	// Default separator is comma
+	if (separator === undefined) {
+		separator = ',';
+	}
+
 	if (stage === 1) {
-		// Initialize: skip null values
+		// Initialize: create array to collect values
+		// Store as object with values array and metadata
 		if (v === null || v === undefined) {
-			return null;
+			return {values: [], separator: separator, orderDirection: orderDirection};
 		}
-		return '' + v;
+		return {values: [v], separator: separator, orderDirection: orderDirection};
 	} else if (stage === 2) {
-		// Accumulate: skip null values
+		// Accumulate: add to values array, skip null values
 		if (v === null || v === undefined) {
 			return s;
 		}
-		// If accumulator is null/undefined, start with current value
+		// If accumulator is null/undefined, initialize it
 		if (s === null || s === undefined) {
-			return '' + v;
+			return {values: [v], separator: separator, orderDirection: orderDirection};
 		}
-		s += ',' + v;
+		// Handle both old string format (for backwards compatibility) and new object format
+		if (typeof s === 'string') {
+			// Old format - convert to new format
+			s = {values: s.split(','), separator: ',', orderDirection: undefined};
+		}
+		s.values.push(v);
 		return s;
+	} else {
+		// Stage 3 (or final): sort if needed and join
+		if (s === null || s === undefined) {
+			return undefined;
+		}
+		// Handle both old string format and new object format
+		if (typeof s === 'string') {
+			return s; // Already formatted
+		}
+
+		let values = s.values;
+
+		// If no values were collected (all nulls), return undefined
+		if (values.length === 0) {
+			return undefined;
+		}
+
+		// Sort if orderDirection is provided (check for actual undefined, not the string 'undefined')
+		if (s.orderDirection && s.orderDirection !== undefined) {
+			let ascending = s.orderDirection === 'ASC';
+			values = values.slice().sort((a, b) => {
+				if (a === b) return 0;
+				if (a === null || a === undefined) return 1;
+				if (b === null || b === undefined) return -1;
+				if (typeof a === 'string' && typeof b === 'string') {
+					return ascending ? a.localeCompare(b) : b.localeCompare(a);
+				}
+				// For numbers and other types - add parentheses for clarity
+				return ascending ? (a < b ? -1 : 1) : b < a ? -1 : 1;
+			});
+		}
+
+		return values.join(s.separator);
 	}
-	return s;
 };
 
 alasql.aggr.median = alasql.aggr.MEDIAN = function (v, s, stage) {
@@ -319,7 +369,7 @@ alasql.aggr.median = alasql.aggr.MEDIAN = function (v, s, stage) {
 	}
 
 	if (!s.length) {
-		return null;
+		return undefined;
 	}
 
 	let r = s.sort((a, b) => {

src/84from.js

@@ -256,6 +256,16 @@ alasql.from.CSV = function (contents, opts, cb, idx, query) {
 	alasql.utils.extend(opt, opts);
 	var res;
 	var hs = [];
+	// Determine once whether to auto-convert: not raw mode and not SELECT INTO
+	const shouldAutoConvert = !opt.raw && !query?.intofns;
+
+	function potentialAutoConvert(val) {
+		if (shouldAutoConvert && val !== undefined && val.length !== 0 && val == +val) {
+			return +val;
+		}
+		return val;
+	}
+
 	function parseText(text) {
 		var delimiterCode = opt.separator.charCodeAt(0);
 		var quoteCode = opt.quote.charCodeAt(0);
@@ -332,25 +342,22 @@ alasql.from.CSV = function (contents, opts, cb, idx, query) {
 						hs = opt.headers;
 						var r = {};
 						hs.forEach(function (h, idx) {
-							r[h] = a[idx];
-							// Keep as string - type conversion happens at INSERT time based on column definitions
+							r[h] = potentialAutoConvert(a[idx]);
 						});
 						rows.push(r);
 					}
 				} else {
 					var r = {};
 					hs.forEach(function (h, idx) {
-						r[h] = a[idx];
-						// Keep as string - type conversion happens at INSERT time based on column definitions
+						r[h] = potentialAutoConvert(a[idx]);
 					});
 					rows.push(r);
 				}
 				n++;
 			} else {
 				var r = {};
-				// Keep as string - type conversion happens at INSERT time based on column definitions
 				a.forEach(function (v, idx) {
-					r[idx] = a[idx];
+					r[idx] = potentialAutoConvert(a[idx]);
 				});
 				rows.push(r);
 			}

src/alasqlparser.jison

@@ -144,6 +144,7 @@ DATABASE(S)?									return 'DATABASE'
 'GO'                                      		return 'GO'
 'GRAPH'                                      	return 'GRAPH'
 'GROUP'                                      	return 'GROUP'
+'GROUP_CONCAT'                                  return 'GROUP_CONCAT'
 'GROUPING'                                     	return 'GROUPING'
 'HAVING'                                        return 'HAVING'
 /*'HELP'											return 'HELP'*/
@@ -233,6 +234,7 @@ SCHEMA(S)?                                      return 'DATABASE'
 'SEARCH'                                        return 'SEARCH'
 
 'SEMI'                                        	return 'SEMI'
+'SEPARATOR'                                     return 'SEPARATOR'
 SET 	                                       	return 'SET'
 SETS                                        	return 'SET'
 'SHOW'                                        	return 'SHOW'
@@ -365,6 +367,8 @@ Literal
 		{ $$ = $1.toLowerCase(); }
 	| CLOSE
 		{ $$ = $1.toLowerCase(); }
+	| SEPARATOR
+		{ $$ = $1.toLowerCase(); }
 	| error NonReserved
 		{ $$ = $2.toLowerCase() }
 	;
@@ -1406,6 +1410,10 @@ AggrValue
 	| Aggregator LPAR ALL Expression RPAR OverClause
 		{ $$ = new yy.AggrValue({aggregatorid: $1.toUpperCase(), expression: $4,
 		 over:$6}); }
+	| GROUP_CONCAT LPAR Expression GroupConcatOrderClause GroupConcatSeparatorClause RPAR
+		{ $$ = new yy.AggrValue({aggregatorid: 'REDUCE', funcid: 'GROUP_CONCAT', expression: $3, order: $4, separator: $5}); }
+	| GROUP_CONCAT LPAR DISTINCT Expression GroupConcatOrderClause GroupConcatSeparatorClause RPAR
+		{ $$ = new yy.AggrValue({aggregatorid: 'REDUCE', funcid: 'GROUP_CONCAT', expression: $4, distinct: true, order: $5, separator: $6}); }
 	;
 
 OverClause
@@ -1422,6 +1430,26 @@ OverOrderByClause
 	: ORDER BY OrderExpressionsList
 		{ $$ = {order:$3}; }
 	;
+
+GroupConcatOrderClause
+	:
+		{ $$ = undefined; }
+	| ORDER BY OrderExpressionsList
+		{ $$ = $3; }
+	;
+
+GroupConcatSeparatorClause
+	:
+		{ $$ = undefined; }
+	| SEPARATOR STRING
+		{ 
+			var str = $2.substring(1, $2.length-1);
+			// Process common escape sequences
+			str = str.replace(/\\n/g, '\n').replace(/\\t/g, '\t').replace(/\\r/g, '\r').replace(/\\\\/g, '\\');
+			$$ = str;
+		}
+	;
+
 Aggregator
 	: SUM { $$ = "SUM"; }
 	| TOTAL { $$ = "TOTAL"; }
@@ -1433,6 +1461,7 @@ Aggregator
 	| LAST { $$ = "LAST"; }
 	| AGGR { $$ = "AGGR"; }
 	| ARRAY { $$ = "ARRAY"; }
+	| GROUP_CONCAT { $$ = "GROUP_CONCAT"; }
 /*	| REDUCE { $$ = "REDUCE"; } */
 	;
 
@@ -3380,6 +3409,7 @@ NonReserved
 	|SECURITY
 	|SELECTIVE
 	|SELF
+	|SEPARATOR
 	|SEQUENCE
 	|SERIALIZABLE
 	|SERVER
@@ -3658,6 +3688,7 @@ var nonReserved = ["A"
 	,"SECURITY"
 	,"SELECTIVE"
 	,"SELF"
+	,"SEPARATOR"
 	,"SEQUENCE"
 	,"SERIALIZABLE"
 	,"SERVER"