feat: implement groupLimit for relationship preloading

groupLimit allows adding limit to a group of preloaded relationships vs
the overall limit. A classic example of this is, fetch 10 users and preload
5 posts for each user

.env

@@ -2,10 +2,15 @@ DB=pg
 DB_NAME=lucid
 
 MYSQL_HOST=mysql
-MYSQL_PORT=3306
+MYSQL_PORT=3300
 MYSQL_USER=virk
 MYSQL_PASSWORD=password
 
+MYSQL_LEGACY_HOST=mysql_legacy
+MYSQL_LEGACY_PORT=3306
+MYSQL_LEGACY_USER=virk
+MYSQL_LEGACY_PASSWORD=password
+
 PG_HOST=pg
 PG_PORT=5432
 PG_USER=virk

adonis-typings/database.ts

@@ -39,6 +39,7 @@ declare module '@ioc:Adonis/Lucid/Database' {
    */
   export interface DialectContract {
     readonly name: 'mssql' | 'mysql' | 'oracledb' | 'postgres' | 'redshift' | 'sqlite3'
+    readonly version?: string
     readonly supportsAdvisoryLocks: boolean
     readonly dateTimeFormat: string
     getAllTables (schemas?: string[]): Promise<string[]>

adonis-typings/relations.ts

@@ -181,7 +181,7 @@ declare module '@ioc:Adonis/Lucid/Relations' {
       HasManyRelationContract<ParentModel, RelatedModel>,
       RelatedModel
     >
-    builder: RelationQueryBuilderContract<RelatedModel, any>,
+    builder: HasManyQueryBuilderContract<RelatedModel, any>,
   }
 
   /**
@@ -232,7 +232,7 @@ declare module '@ioc:Adonis/Lucid/Relations' {
       HasManyThroughRelationContract<ParentModel, RelatedModel>,
       RelatedModel
     >,
-    builder: RelationQueryBuilderContract<RelatedModel, any>,
+    builder: HasManyThroughQueryBuilderContract<RelatedModel, any>,
   }
 
   /**
@@ -747,6 +747,28 @@ declare module '@ioc:Adonis/Lucid/Relations' {
     selectRelationKeys (): this
   }
 
+  /**
+   * Has many query builder contract
+   */
+  export interface HasManyQueryBuilderContract<
+    Related extends LucidModel,
+    Result extends any
+  > extends RelationQueryBuilderContract<Related, Result> {
+    groupLimit (limit: number): this
+    groupOrderBy (column: string, direction?: 'asc' | 'desc'): this
+  }
+
+  /**
+   * Has many query through builder contract
+   */
+  export interface HasManyThroughQueryBuilderContract<
+    Related extends LucidModel,
+    Result extends any
+  > extends RelationQueryBuilderContract<Related, Result> {
+    groupLimit (limit: number): this
+    groupOrderBy (column: string, direction?: 'asc' | 'desc'): this
+  }
+
   /**
    * Possible signatures for adding a where clause
    */
@@ -773,6 +795,9 @@ declare module '@ioc:Adonis/Lucid/Relations' {
     Related extends LucidModel,
     Result extends any,
   > extends RelationQueryBuilderContract<Related, Result> {
+    groupLimit (limit: number): this
+    groupOrderBy (column: string, direction?: 'asc' | 'desc'): this
+
     pivotColumns (columns: string[]): this
     isPivotOnlyQuery: boolean
 

docker-compose-test.yml

@@ -5,6 +5,7 @@ services:
       context: .
       target: build-deps
     links:
+      - mysql_legacy
       - mysql
       - pg
       - mssql

docker-compose.yml

@@ -1,6 +1,6 @@
 version: '3.4'
 services:
-  mysql:
+  mysql_legacy:
     image: mysql:5.7
     restart: always
     environment:
@@ -12,6 +12,19 @@ services:
       - '3306:3306'
     expose:
       - '3306'
+  mysql:
+    image: mysql:8.0
+    restart: always
+    command: --default-authentication-plugin=mysql_native_password --sync_binlog=0 --innodb_doublewrite=OFF  --innodb-flush-log-at-trx-commit=0 --innodb-flush-method=nosync
+    environment:
+      MYSQL_DATABASE: lucid
+      MYSQL_USER: virk
+      MYSQL_PASSWORD: password
+      MYSQL_ROOT_PASSWORD: password
+    ports:
+      - '3300:3306'
+    expose:
+      - '3300'
   pg:
     image: postgres:11
     restart: always

japaFile.js

@@ -1,5 +1,6 @@
-process.env.TS_NODE_FILES = 'true'
-require('ts-node/register')
+require('ts-node').register({
+  files: true,
+})
 
 const { configure } = require('japa')
 configure({

package.json

@@ -16,9 +16,10 @@
     "pretest": "npm run lint",
     "test:sqlite": "DB=sqlite node japaFile.js",
     "test:mysql": "DB=mysql node japaFile.js",
+    "test:mysql_legacy": "DB=mysql_legacy node japaFile.js",
     "test:mssql": "DB=mssql node japaFile.js",
     "test:pg": "DB=pg node japaFile.js",
-    "test:docker": "npm run test:sqlite && npm run test:mysql && npm run test:pg && npm run test:mssql",
+    "test:docker": "npm run test:sqlite && npm run test:mysql && npm run test:mysql_legacy && npm run test:pg && npm run test:mssql",
     "test": "docker-compose -f docker-compose.yml -f docker-compose-test.yml build && docker-compose -f docker-compose.yml -f docker-compose-test.yml run --rm test",
     "lint": "eslint . --ext=.ts",
     "clean": "del build",

src/Database/QueryBuilder/Chainable.ts

@@ -211,7 +211,7 @@ export abstract class Chainable extends Macroable implements ChainableContract {
    * use the last selected table
    */
   public from (table: any): this {
-    this.knexQuery.from(this.transformCallback(table))
+    this.knexQuery.from(this.transformValue(table))
     return this
   }
 

src/Dialects/Mssql.ts

@@ -16,6 +16,12 @@ export class MssqlDialect implements DialectContract {
   public readonly name = 'mssql'
   public readonly supportsAdvisoryLocks = false
 
+  /**
+   * Reference to the database version. Knex.js fetches the version after
+   * the first database query, so it will be set to undefined initially
+   */
+  public readonly version = this.client.getReadClient()['context']['client'].version
+
   /**
    * The default format for datetime column. The date formats is
    * valid for luxon date parsing library
@@ -32,7 +38,7 @@ export class MssqlDialect implements DialectContract {
     const tables = await this.client
       .query()
       .from('information_schema.tables')
-      .select('table_name')
+      .select('table_name as table_name')
       .where('table_type', 'BASE TABLE')
       .where('table_catalog', new RawBuilder('DB_NAME()'))
       .whereNot('table_name', 'like', 'spt_%')

src/Dialects/Mysql.ts

@@ -16,6 +16,12 @@ export class MysqlDialect implements DialectContract {
   public readonly name = 'mysql'
   public readonly supportsAdvisoryLocks = true
 
+  /**
+   * Reference to the database version. Knex.js fetches the version after
+   * the first database query, so it will be set to undefined initially
+   */
+  public readonly version = this.client.getReadClient()['context']['client'].version
+
   /**
    * The default format for datetime column. The date formats is
    * valid for luxon date parsing library
@@ -55,9 +61,8 @@ export class MysqlDialect implements DialectContract {
     const tables = await this.client
       .query()
       .from('information_schema.tables')
-      .select('table_name')
+      .select('table_name as table_name')
       .where('TABLE_TYPE', 'BASE TABLE')
-      .debug(true)
       .where('table_schema', new RawBuilder('database()'))
       .orderBy('table_name', 'asc')
 

src/Dialects/Oracle.ts

@@ -15,6 +15,12 @@ export class OracleDialect implements DialectContract {
   public readonly name = 'oracledb'
   public readonly supportsAdvisoryLocks = false
 
+  /**
+   * Reference to the database version. Knex.js fetches the version after
+   * the first database query, so it will be set to undefined initially
+   */
+  public readonly version = this.client.getReadClient()['context']['client'].version
+
   /**
    * The default format for datetime column. The date formats is
    * valid for luxon date parsing library

src/Dialects/Pg.ts

@@ -15,6 +15,12 @@ export class PgDialect implements DialectContract {
   public readonly name = 'postgres'
   public readonly supportsAdvisoryLocks = true
 
+  /**
+   * Reference to the database version. Knex.js fetches the version after
+   * the first database query, so it will be set to undefined initially
+   */
+  public readonly version = this.client.getReadClient()['context']['client'].version
+
   /**
    * The default format for datetime column. The date formats is
    * valid for luxon date parsing library
@@ -31,11 +37,11 @@ export class PgDialect implements DialectContract {
     const tables = await this.client
       .query()
       .from('pg_catalog.pg_tables')
-      .select('tablename')
+      .select('tablename as table_name')
       .whereIn('schemaname', schemas)
       .orderBy('tablename', 'asc')
 
-    return tables.map(({ tablename }) => tablename)
+    return tables.map(({ table_name }) => table_name)
   }
 
   /**

src/Dialects/Redshift.ts

@@ -15,6 +15,12 @@ export class RedshiftDialect implements DialectContract {
   public readonly name = 'redshift'
   public readonly supportsAdvisoryLocks = false
 
+  /**
+   * Reference to the database version. Knex.js fetches the version after
+   * the first database query, so it will be set to undefined initially
+   */
+  public readonly version = this.client.getReadClient()['context']['client'].version
+
   /**
    * The default format for datetime column. The date formats is
    * valid for luxon date parsing library
@@ -33,11 +39,11 @@ export class RedshiftDialect implements DialectContract {
     const tables = await this.client
       .query()
       .from('pg_catalog.pg_tables')
-      .select('tablename')
+      .select('tablename as table_name')
       .whereIn('schemaname', schemas)
       .orderBy('tablename', 'asc')
 
-    return tables.map(({ tablename }) => tablename)
+    return tables.map(({ table_name }) => table_name)
   }
 
   /**

src/Dialects/Sqlite.ts

@@ -15,6 +15,12 @@ export class SqliteDialect implements DialectContract {
   public readonly name = 'sqlite3'
   public readonly supportsAdvisoryLocks = false
 
+  /**
+   * Reference to the database version. Knex.js fetches the version after
+   * the first database query, so it will be set to undefined initially
+   */
+  public readonly version = this.client.getReadClient()['context']['client'].version
+
   /**
    * The default format for datetime column. The date formats is
    * valid for luxon date parsing library
@@ -31,12 +37,12 @@ export class SqliteDialect implements DialectContract {
     const tables = await this.client
       .query()
       .from('sqlite_master')
-      .select('name')
+      .select('name as table_name')
       .where('type', 'table')
       .whereNot('name', 'like', 'sqlite_%')
       .orderBy('name', 'asc')
 
-    return tables.map(({ name }) => name)
+    return tables.map(({ table_name }) => table_name)
   }
 
   /**

src/Orm/Relations/Base/QueryBuilder.ts

@@ -8,9 +8,9 @@
 */
 
 import knex from 'knex'
-import { LucidModel, LucidRow } from '@ioc:Adonis/Lucid/Model'
 import { QueryClientContract } from '@ioc:Adonis/Lucid/Database'
 import { DBQueryCallback } from '@ioc:Adonis/Lucid/DatabaseQueryBuilder'
+import { LucidModel, LucidRow, ModelQueryBuilderContract } from '@ioc:Adonis/Lucid/Model'
 import { RelationQueryBuilderContract, RelationshipsContract } from '@ioc:Adonis/Lucid/Relations'
 
 import { ModelQueryBuilder } from '../../QueryBuilder'
@@ -19,9 +19,14 @@ import { ModelQueryBuilder } from '../../QueryBuilder'
  * Base query builder for ORM Relationships
  */
 export abstract class BaseQueryBuilder extends ModelQueryBuilder implements RelationQueryBuilderContract<
-LucidModel,
-LucidRow
+  LucidModel,
+  LucidRow
 > {
+  /**
+   * Eager constraints
+   */
+  protected groupConstraints: { limit?: number, orderBy?: string } = {}
+
   /**
    * A flag to know, if query builder is instantiated for
    * eager loading or not.
@@ -37,6 +42,13 @@ LucidRow
     super(builder, relation.relatedModel(), client, dbCallback)
   }
 
+  /**
+   * Returns the selected columns
+   */
+  protected getSelectedColumns (): undefined | { grouping: 'columns', value: any[] } {
+    return this.knexQuery['_statements'].find(({ grouping }) => grouping === 'columns')
+  }
+
   /**
    * Returns the profiler action. Protected, since the class is extended
    * by relationships
@@ -67,6 +79,12 @@ LucidRow
    */
   protected abstract applyConstraints (): void
 
+  /**
+   * Must be implemented by relationships to return query which
+   * handles the limit with eagerloading.
+   */
+  protected abstract getGroupLimitQuery (): never | ModelQueryBuilderContract<LucidModel>
+
   /**
    * Returns the name of the query action. Used mainly for
    * raising descriptive errors
@@ -88,8 +106,7 @@ LucidRow
    * Selects the relation keys. Invoked by the preloader
    */
   public selectRelationKeys (): this {
-    const knexQuery = this.knexQuery
-    const columns = knexQuery['_statements'].find(({ grouping }) => grouping === 'columns')
+    const columns = this.getSelectedColumns()
 
     /**
      * No columns have been defined, we will let knex do it's job by
@@ -112,6 +129,22 @@ LucidRow
     return this
   }
 
+  /**
+   * Define the group limit
+   */
+  public groupLimit (limit: number): this {
+    this.groupConstraints.limit = limit
+    return this
+  }
+
+  /**
+   * Define the group limit
+   */
+  public groupOrderBy (column: string, direction?: 'asc' | 'desc'): this {
+    this.groupConstraints.orderBy = direction ? `${this.resolveKey(column)} ${direction}` : column
+    return this
+  }
+
   /**
    * Get query sql
    */
@@ -125,6 +158,6 @@ LucidRow
    */
   public exec () {
     this.applyConstraints()
-    return super.exec()
+    return this.groupConstraints.limit ? this.getGroupLimitQuery().exec() : super.exec()
   }
 }

src/Orm/Relations/BelongsTo/QueryBuilder.ts

@@ -144,4 +144,11 @@ export class BelongsToQueryBuilder extends BaseQueryBuilder {
   public paginate (): Promise<any> {
     throw new Error(`Cannot paginate a belongsTo relationship "(${this.relation.relationName})"`)
   }
+
+  /**
+   * Dis-allow belongsTo group query limit
+   */
+  public getGroupLimitQuery (): never {
+    throw new Error(`Cannot apply groupLimit or groupOrderBy on a belongsTo relationship "(${this.relation.relationName})"`)
+  }
 }