JS computed columns (#270)

also, change `returnType` type to be `undefined` by default. (#299)

.changeset/silly-waves-itch.md

@@ -0,0 +1,9 @@
+---
+'pqb': minor
+'orchid-core': minor
+'orchid-orm': minor
+---
+
+JS computed columns;
+
+Change `returnType` type to be `undefined` by default.

BREAKING_CHANGES.md

@@ -1,5 +1,23 @@
 # Breaking changes
 
+## orchid-orm 1.31.0
+
+Computed columns change, see the [docs](https://orchid-orm.netlify.app/guide/computed-columns.html).
+
+For SQL computed columns:
+
+```ts
+// before
+computed = this.setComputed({
+ fullName: (q) => q.sql`...`.type((t) => t.string()),
+});
+
+// after
+computed = this.setComputed((q) => ({
+ fullName: q.sql`...`.type((t) => t.string()),
+}));
+```
+
 ## orchid-orm 1.30.0
 
 The `text` column type no longer accepts `min` and `max` params.

docs/src/.vitepress/config.ts

@@ -78,6 +78,10 @@ export default {
  text: 'JSON functions',
  link: '/guide/json',
  },
+ {
+ text: 'Computed columns',
+ link: '/guide/computed-columns',
+ },
  {
  text: 'Window functions',
  link: '/guide/window',

docs/src/guide/computed-columns.md

@@ -0,0 +1,212 @@
+# Computed columns
+
+OrchidORM supports defining columns that are calculated on the fly,
+either by injecting SQL into a `SELECT` statement, or by computing values in runtime on JS side.
+
+Note that unlike regular columns, computed columns are not selected by default.
+
+Alternatively, you can add a generated column in the migration (see [generated](/guide/migration-column-methods#generated-column)),
+such column will persist in the database.
+
+## SQL computed column
+
+SQL computed column is going to unwrap into the given SQL when selecting it from the table.
+
+In the following example, selecting `fullName` will unwrap into `"firstName" || ' ' || "lastName"` SQL:
+
+```ts
+export class UserTable extends BaseTable {
+ readonly table = 'user';
+ columns = this.setColumns((t) => ({
+ id: t.identity().primaryKey(),
+ firstName: t.string(),
+ lastName: t.string(),
+ }));
+
+ computed = this.setComputed((q) => ({
+ fullName: q.sql`${q.column('firstName')} || ' ' || ${q.column(
+ 'lastName',
+ )}`.type((t) => t.string()),
+ randomizedName: q
+ .sql(() => q.sql`${Math.random()} ${q.column('firstName')}`)
+ .type((t) => t.string()),
+ }));
+}
+```
+
+`randomizedName` in the example is defined with `` q.sql(() => q.sql`...`) `` syntax that makes it dynamic,
+so that a new random value will be selected for every query.
+
+Such can be column can be selected, can be used for filtering and ordering, available in nested sub-queries.
+
+```ts
+// select all columns + the computed
+db.user.select('*', 'fullName')
+
+// use in nested select
+db.chat.find(id).select({
+ messages: (q) => q.messages.select({
+ // select fullName for a single row
+ sender: (q) => q.sender.select('fullName')
+ // `pluck` will load a flat array of values
+ receipients: (q) =>
+ q.receipients
+ .pluck('fullName')
+ // works for filtering
+ .where({ fullName: { startsWith: 'x' } })
+ // works for ordering
+ .order('fullName'),
+ })
+})
+
+// can be selected for a joined table
+db.post.join('author').select('author.fullName')
+
+// can be returned from `insert`, `create`, `update`, `delete`, `upsert`
+db.user.select('fullName').insert(data)
+```
+
+## JS runtime computed
+
+Define a runtime computed column to compute values after loading results.
+
+Unlike SQL computed columns, these columns aren't suitable for filtering or ordering records, they only can be used in selects.
+
+```ts
+export class UserTable extends BaseTable {
+ readonly table = 'user';
+ columns = this.setColumns((t) => ({
+ id: t.identity().primaryKey(),
+ firstName: t.string(),
+ lastName: t.string(),
+ }));
+
+ computed = this.setComputed((q) => ({
+ fullName: q.computeAtRuntime(
+ // define columns that it depends on
+ ['firstName', 'lastName'],
+ // only columns defined above are available in the callback
+ (record) => `${record.firstName} ${record.lastName}`,
+ ),
+ }));
+}
+```
+
+The runtime computed column is available in all kinds of selections.
+
+It will automatically select dependencies, if they weren't selected,
+and will dispose dependencies after computing a value if they weren't selected.
+
+```ts
+const record = await db.user.select('firstName', 'fullName');
+record.firstName; // was selected
+record.fullName; // was computed
+record.lastName; // TS error: it was selected but then disposed
+
+db.char.find(id).select({
+ messages: (q) => q.messages.select({
+ // select fullName for a single row
+ sender: (q) => q.sender.select('fullName')
+ // `pluck` will collect a flat array of values
+ receipients: (q) => q.receipients.pluck('fullName')
+ })
+})
+
+// can be selected for a joined table
+db.post.join('author').select('author.fullName')
+
+// can be returned from `insert`, `create`, `update`, `delete`, `upsert`
+db.user.select('fullName').insert(data)
+```
+
+## Async computed columns
+
+Asynchronously fetching data for records one-by-one would take a lot of loading time,
+it's much better to load data in batches.
+
+```ts
+interface WeatherData {
+ country: string;
+ city: string;
+ weatherInfo: SomeStructure;
+}
+
+export class UserTable extends BaseTable {
+ readonly table = 'user';
+ columns = this.setColumns((t) => ({
+ id: t.identity().primaryKey(),
+ country: t.string(),
+ city: t.string(),
+ }));
+
+ computed = this.setComputed((q) => ({
+ weather: q.computeBatchAtRuntime(
+ // define columns that it depends on
+ ['country', 'city'],
+ // load weather data for all users using a single fetch
+ async (users): Promise<(SomeStructure | undefined)[]> => {
+ // to not query the same location twice
+ const uniqueLocations = new Set(
+ users.map((user) => `${user.country} ${user.city}`),
+ );
+
+ // fetch data for all locations at once
+ const weatherData: WeatherData[] = await fetchWeatherData({
+ location: [...uniqueLocations],
+ });
+
+ // return array with weather data for every user
+ return users.map(
+ (user) =>
+ weatherData.find(
+ (wd) => wd.country === user.country && wd.city === user.city,
+ )?.weatherInfo,
+ );
+ },
+ ),
+ }));
+}
+```
+
+`computeBatchAtRuntime` can also take a synchronous function.
+
+From a querying perspective, there is no difference from a [computeAtRuntime](#js-runtime-computed) column,
+it works and acts in the same way.
+
+```ts
+db.user.select('*', 'weather');
+
+// a city can have millions of people,
+// but the weather is loaded just once
+db.city.find(id).select({
+ users: (q) => q.users.select('name', 'weather'),
+});
+```
+
+Only a single batch of records is processed even when loading a nested query.
+
+Let's say we have 10 countries, every country has 10 cities, with 100 users in each.
+
+The `weather` computed column will be called just once with 10_000 of records.
+
+```ts
+db.country.select({
+ cities: (q) =>
+ q.cities.select({
+ users: (q) => q.users.select('name', 'weather'),
+ }),
+});
+```
+
+A city may have a mayor, but that's not always the case.
+Null records are omitted when passing data to a computed column.
+
+```ts
+db.country.select({
+ cities: (q) =>
+ q.cities.select({
+ // city hasOne mayor, not required
+ mayor: (q) => q.mayor.select('name', 'weather').,
+ }),
+});
+```

docs/src/guide/orm-and-query-builder.md

@@ -852,11 +852,6 @@ await db.some.unscope('default');
 
 [//]: # 'has JSDoc'
 
-You can add a generated column in the migration (see [generated](/guide/migration-column-methods#generated-column)),
-such column will persist in the database, it can be indexed.
-
-Or you can add a computed column on the ORM level, without adding it to the database, in such a way:
-
 ```ts
 import { BaseTable } from './baseTable';
 

packages/core/src/query.ts

@@ -3,6 +3,7 @@ import { TransactionState } from './adapter';
 import {
  EmptyObject,
  FnUnknownToUnknown,
+ MaybePromise,
  RecordKeyTrue,
  RecordUnknown,
 } from './utils';
@@ -77,6 +78,7 @@ export type CoreQueryScopes<Keys extends string = string> = {
 };
 
 export type QueryReturnType =
+ | undefined
  | 'all'
  | 'one'
  | 'oneOrThrow'
@@ -186,11 +188,25 @@ export const getValueKey = Symbol('get');
 // function to parse a single column after loading the data
 export type ColumnParser = FnUnknownToUnknown;
 
-// functions to parse columns after loading the data
+// To parse all returned rows. Unlike column parser, can return a promise.
+export interface BatchParser {
+ path: string[];
+ fn: (path: string[], queryResult: { rows: unknown[] }) => MaybePromise<void>;
+}
+
+// set of value parsers
 // key is a name of a selected column,
 // or it can be a `getValueKey` to parse single values requested by the `.get()`, `.count()`, or similar methods
 export type ColumnsParsers = { [K in string | getValueKey]?: ColumnParser };
 
+// set of batch parsers
+// is only triggered when loading all,
+// or when using `hookSelect` or computed columns that convert response to `all` internally.
+// key is a name of a selected column,
+// or it can be a `getValueKey` to parse single values requested by the `.get()`, `.count()`, or similar methods
+export type BatchParsers = BatchParser[];
+
+// result transformer: function for `transform`, object for `map`
 export type QueryDataTransform =
  | FnUnknownToUnknown
  | { map: FnUnknownToUnknown };
@@ -246,7 +262,7 @@ export const applyTransforms = (
 ): unknown => {
  for (const fn of fns) {
  if ('map' in fn) {
- if (returnType === 'all') {
+ if (!returnType || returnType === 'all') {
  result = (result as unknown[]).map(fn.map);
  } else {
  result = fn.map(result);

packages/core/src/raw.ts

@@ -64,13 +64,13 @@ export const isTemplateLiteralArgs = (
 // Argument type for `sql` function.
 // It can take a template literal, an object `{ raw: string, values?: Record<string, unknown> }`,
 // or a function to build SQL lazily.
-export type SQLArgs = StaticSQLArgs | [DynamicSQLArg];
+export type SQLArgs = StaticSQLArgs | [DynamicSQLArg<QueryColumn>];
 
 // Function for sql method to build SQL lazily (dynamically).
 // May be used for computed column to build a different SQL in different executions.
-export type DynamicSQLArg = (
- sql: (...args: StaticSQLArgs) => Expression,
-) => Expression;
+export type DynamicSQLArg<T extends QueryColumn> = (
+ sql: (...args: StaticSQLArgs) => Expression<T>,
+) => Expression<T>;
 
 // SQL arguments for a non-lazy SQL expression.
 export type StaticSQLArgs =

packages/core/src/utils.ts

@@ -353,13 +353,6 @@ export const callWithThis = function <T, R>(this: T, cb: (arg: T) => R): R {
  return cb(this);
 };
 
-export const cloneInstance = <T>(instance: T): T => {
- return Object.assign(
- Object.create(Object.getPrototypeOf(instance)),
- instance,
- );
-};
-
 export const pick = <T, Keys extends keyof T>(
  obj: T,
  keys: Keys[],