adding support for DISTINCT ON for postgres DB

pkg/dbsql/crud.go

@@ -1,4 +1,4 @@
-// Copyright © 2024 Kaleido, Inc.
+// Copyright © 2024 - 2026 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -785,13 +785,48 @@ func (c *CrudBase[T]) getManyScoped(ctx context.Context, tableFrom string, fi *f
 	if err != nil {
 		return nil, nil, err
 	}
+
+	// Apply ReadQueryModifier first (before wrapping with DISTINCT ON)
 	if c.ReadQueryModifier != nil {
 		if query, err = c.ReadQueryModifier(query); err != nil {
 			return nil, nil, err
 		}
 	}
 
-	rows, tx, err := c.DB.Query(ctx, c.Table, query)
+	// Apply placeholder format to the query
+	query = query.PlaceholderFormat(c.DB.features.PlaceholderFormat)
+
+	// Apply DISTINCT ON wrapper for PostgreSQL if needed
+	var finalQuery sq.Sqlizer = query
+	if len(fi.DistinctOn) > 0 {
+		if c.DB.provider == nil || c.DB.provider.Name() != "postgres" {
+			providerName := "unknown"
+			if c.DB.provider != nil {
+				providerName = c.DB.provider.Name()
+			}
+			return nil, nil, i18n.NewError(ctx, i18n.MsgDistinctOnNotSupported, providerName)
+		}
+		distinctOnFields := make([]string, len(fi.DistinctOn))
+		for i, do := range fi.DistinctOn {
+			// Map the field name using the FilterFieldMap
+			mappedField := do
+			if c.FilterFieldMap != nil {
+				if mf, ok := c.FilterFieldMap[do]; ok {
+					mappedField = mf
+				}
+			}
+			if c.ReadTableAlias != "" && !strings.Contains(mappedField, ".") {
+				mappedField = fmt.Sprintf("%s.%s", c.ReadTableAlias, mappedField)
+			}
+			distinctOnFields[i] = mappedField
+		}
+		finalQuery = &distinctOnSqlizer{
+			inner:      query,
+			distinctOn: distinctOnFields,
+		}
+	}
+
+	rows, tx, err := c.DB.RunAsQueryTx(ctx, c.Table, nil, finalQuery)
 	if err != nil {
 		return nil, nil, err
 	}

pkg/dbsql/crud_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 Kaleido, Inc.
+// Copyright © 2023 - 2026 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -112,6 +112,7 @@ var LinkableQueryFactory = &ffapi.QueryFields{
 	"id":      &ffapi.UUIDField{},
 	"created": &ffapi.TimeField{},
 	"updated": &ffapi.TimeField{},
+	"ns":      &ffapi.StringField{},
 	"crud":    &ffapi.UUIDField{},
 }
 
@@ -1525,3 +1526,164 @@ func TestCustomIDColumn(t *testing.T) {
 	}
 	tc.Validate()
 }
+
+// postgresMockProvider wraps MockProvider to return "postgres" as the name for DISTINCT ON testing
+type postgresMockProvider struct {
+	*MockProvider
+}
+
+func (p *postgresMockProvider) Name() string {
+	return "postgres"
+}
+
+func newPostgresMockProvider() (*postgresMockProvider, sqlmock.Sqlmock) {
+	mp := NewMockProvider()
+	pgmp := &postgresMockProvider{MockProvider: mp}
+	_ = pgmp.Database.Init(context.Background(), pgmp, pgmp.config)
+	return pgmp, pgmp.mdb
+}
+
+func TestGetManyWithDistinctOnPostgreSQL(t *testing.T) {
+	db, mock := newPostgresMockProvider()
+	tc := newCRUDCollection(&db.Database, "ns1")
+
+	// Test DISTINCT ON with a single field
+	filter := CRUDableQueryFactory.NewFilter(context.Background()).
+		DistinctOn("ns").
+		And()
+
+	// Expect query with DISTINCT ON
+	mock.ExpectQuery(`SELECT DISTINCT ON \(.*ns.*\)`).
+		WillReturnRows(sqlmock.NewRows([]string{"seq", "id", "created", "updated", "ns", "name", "field1", "field2", "field3", "field4", "field5", "field6"}).
+			AddRow(1, fftypes.NewUUID().String(), fftypes.Now().String(), fftypes.Now().String(), "ns1", "test", "test", nil, nil, nil, nil, nil))
+
+	results, _, err := tc.GetMany(context.Background(), filter)
+	assert.NoError(t, err)
+	assert.NotNil(t, results)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetManyWithDistinctOnMultipleFieldsPostgreSQL(t *testing.T) {
+	db, mock := newPostgresMockProvider()
+	tc := newCRUDCollection(&db.Database, "ns1")
+
+	// Test DISTINCT ON with multiple fields
+	filter := CRUDableQueryFactory.NewFilter(context.Background()).
+		DistinctOn("ns", "name").
+		And()
+
+	// Expect query with DISTINCT ON for multiple fields
+	mock.ExpectQuery(`SELECT DISTINCT ON \(.*ns.*name.*\)`).
+		WillReturnRows(sqlmock.NewRows([]string{"seq", "id", "created", "updated", "ns", "name", "field1", "field2", "field3", "field4", "field5", "field6"}).
+			AddRow(1, fftypes.NewUUID().String(), fftypes.Now().String(), fftypes.Now().String(), "ns1", "test", "test", nil, nil, nil, nil, nil))
+
+	results, _, err := tc.GetMany(context.Background(), filter)
+	assert.NoError(t, err)
+	assert.NotNil(t, results)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetManyWithDistinctOnAndReadTableAliasPostgreSQL(t *testing.T) {
+	db, mock := newPostgresMockProvider()
+	linkables := newLinkableCollection(&db.Database, "ns1")
+
+	// Test DISTINCT ON with ReadTableAlias
+	filter := LinkableQueryFactory.NewFilter(context.Background()).
+		DistinctOn("ns").
+		And()
+
+	// Expect query with DISTINCT ON using table alias
+	// The regex should match: SELECT DISTINCT ON (l.ns) ...
+	mock.ExpectQuery(`SELECT.*DISTINCT ON.*l\.ns`).
+		WillReturnRows(sqlmock.NewRows([]string{"seq", "id", "created", "updated", "ns", "desc", "crud_id", "c.field1", "c.field2", "c.field3"}).
+			AddRow(1, fftypes.NewUUID().String(), fftypes.Now().String(), fftypes.Now().String(), "ns1", "desc", nil, "", nil, nil))
+
+	results, _, err := linkables.GetMany(context.Background(), filter)
+	assert.NoError(t, err)
+	assert.NotNil(t, results)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetManyWithDistinctOnAndReadQueryModifierPostgreSQL(t *testing.T) {
+	db, mock := newPostgresMockProvider()
+	tc := newCRUDCollection(&db.Database, "ns1")
+
+	// Set up a ReadQueryModifier
+	tc.ReadQueryModifier = func(sb sq.SelectBuilder) (sq.SelectBuilder, error) {
+		return sb.Where(sq.Eq{"ns": "ns1"}), nil
+	}
+
+	// Test DISTINCT ON with ReadQueryModifier
+	filter := CRUDableQueryFactory.NewFilter(context.Background()).
+		DistinctOn("ns").
+		And()
+
+	// Expect query with DISTINCT ON - modifier should be applied to inner query before wrapping
+	mock.ExpectQuery(`SELECT DISTINCT ON \(.*ns.*\)`).
+		WillReturnRows(sqlmock.NewRows([]string{"seq", "id", "created", "updated", "ns", "name", "field1", "field2", "field3", "field4", "field5", "field6"}).
+			AddRow(1, fftypes.NewUUID().String(), fftypes.Now().String(), fftypes.Now().String(), "ns1", "test", "test", nil, nil, nil, nil, nil))
+
+	results, _, err := tc.GetMany(context.Background(), filter)
+	assert.NoError(t, err)
+	assert.NotNil(t, results)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetManyWithDistinctOnNonPostgreSQL(t *testing.T) {
+	// Test that DISTINCT ON throws an error for non-PostgreSQL databases
+	db, mock := NewMockProvider().UTInit()
+	tc := newCRUDCollection(&db.Database, "ns1")
+
+	// Test DISTINCT ON with non-postgres provider (should return error)
+	filter := CRUDableQueryFactory.NewFilter(context.Background()).
+		DistinctOn("ns").
+		And()
+
+	// Should return error, no query should be executed
+	_, _, err := tc.GetMany(context.Background(), filter)
+	assert.Error(t, err)
+	assert.Regexp(t, "FF00258", err)
+	assert.Contains(t, err.Error(), "DISTINCT ON is only supported for PostgreSQL")
+	assert.Contains(t, err.Error(), "mockdb")
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetManyWithDistinctOnAndFilterFieldMapPostgreSQL(t *testing.T) {
+	db, mock := newPostgresMockProvider()
+	tc := newCRUDCollection(&db.Database, "ns1")
+
+	// Test DISTINCT ON with field mapping (f1 -> field1)
+	filter := CRUDableQueryFactory.NewFilter(context.Background()).
+		DistinctOn("f1").
+		And()
+
+	// Expect query with DISTINCT ON using mapped field name
+	mock.ExpectQuery(`SELECT DISTINCT ON \(.*field1.*\)`).
+		WillReturnRows(sqlmock.NewRows([]string{"seq", "id", "created", "updated", "ns", "name", "field1", "field2", "field3", "field4", "field5", "field6"}).
+			AddRow(1, fftypes.NewUUID().String(), fftypes.Now().String(), fftypes.Now().String(), "ns1", "test", "test", nil, nil, nil, nil, nil))
+
+	results, _, err := tc.GetMany(context.Background(), filter)
+	assert.NoError(t, err)
+	assert.NotNil(t, results)
+	assert.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetManyWithDistinctOnReadQueryModifierError(t *testing.T) {
+	db, mock := newPostgresMockProvider()
+	tc := newCRUDCollection(&db.Database, "ns1")
+
+	// Set up a ReadQueryModifier that returns an error
+	tc.ReadQueryModifier = func(sb sq.SelectBuilder) (sq.SelectBuilder, error) {
+		return sb, fmt.Errorf("modifier error")
+	}
+
+	// Test DISTINCT ON with ReadQueryModifier error
+	filter := CRUDableQueryFactory.NewFilter(context.Background()).
+		DistinctOn("ns").
+		And()
+
+	_, _, err := tc.GetMany(context.Background(), filter)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "modifier error")
+	assert.NoError(t, mock.ExpectationsWereMet())
+}

pkg/dbsql/database.go

@@ -1,4 +1,4 @@
-// Copyright © 2024 Kaleido, Inc.
+// Copyright © 2024 - 2026 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -20,6 +20,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 
 	sq "github.com/Masterminds/squirrel"
@@ -205,10 +206,39 @@ func (s *Database) QueryTx(ctx context.Context, table string, tx *TXWrapper, q s
 	return s.RunAsQueryTx(ctx, table, tx, q.PlaceholderFormat(s.features.PlaceholderFormat))
 }
 
+// distinctOnSqlizer wraps a SelectBuilder to add PostgreSQL DISTINCT ON support
+type distinctOnSqlizer struct {
+	inner      sq.SelectBuilder
+	distinctOn []string
+}
+
+func (d *distinctOnSqlizer) ToSql() (string, []interface{}, error) {
+	sql, args, err := d.inner.ToSql()
+	if err != nil {
+		return sql, args, err
+	}
+	// Replace SELECT with SELECT DISTINCT ON (fields)
+	if len(d.distinctOn) > 0 {
+		distinctOnClause := fmt.Sprintf("DISTINCT ON (%s) ", strings.Join(d.distinctOn, ", "))
+		sql = strings.Replace(sql, "SELECT ", fmt.Sprintf("SELECT %s", distinctOnClause), 1)
+	}
+	return sql, args, nil
+}
+
 func (s *Database) RunAsQueryTx(ctx context.Context, table string, tx *TXWrapper, q sq.Sqlizer) (*sql.Rows, *TXWrapper, error) {
 
 	l := log.L(ctx)
-	sqlQuery, args, err := q.ToSql()
+
+	// Check if this is a distinctOnSqlizer wrapper
+	var sqlQuery string
+	var args []interface{}
+	var err error
+	if dos, ok := q.(*distinctOnSqlizer); ok {
+		sqlQuery, args, err = dos.ToSql()
+	} else {
+		sqlQuery, args, err = q.ToSql()
+	}
+
 	if err != nil {
 		return nil, tx, i18n.WrapError(ctx, err, i18n.MsgDBQueryBuildFailed)
 	}