Add GPRC methods for the duplicate payments table for the SQL backend

.gitignore

@@ -59,6 +59,9 @@ mobile/*_generated.go
 *.hex
 *.db
 *.bin
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
 
 vendor
 *.idea

cmd/commands/cmd_payments.go

@@ -1525,6 +1525,71 @@
 	return nil
 }
 
+var listPaymentDuplicatesCommand = cli.Command{
+	Name:     "listpaymentduplicates",
+	Category: "Payments",
+	Usage:    "List duplicate payments for a given payment hash.",
+	Hidden:   true,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "payment_hash",
+			Usage: "hex-encoded payment hash to query",
+		},
+	},
+	Action: actionDecorator(listPaymentDuplicates),
+}
+
+func listPaymentDuplicates(ctx *cli.Context) error {
+	if !ctx.IsSet("payment_hash") {
+		return fmt.Errorf("payment_hash is required")
+	}
+
+	hashBytes, err := hex.DecodeString(ctx.String("payment_hash"))
+	if err != nil {
+		return fmt.Errorf("error decoding payment_hash: %w", err)
+	}
+
+	ctxc := getContext()
+	client, cleanUp := getClient(ctx)
+	defer cleanUp()
+
+	req := &lnrpc.ListPaymentDuplicatesRequest{
+		PaymentHash: hashBytes,
+	}
+
+	resp, err := client.ListPaymentDuplicates(ctxc, req)
+	if err != nil {
+		return err
+	}
+
+	printRespJSON(resp)
+	return nil
+}
+
+var listAllPaymentDuplicatesCommand = cli.Command{
+	Name:     "listallpaymentduplicates",
+	Category: "Payments",
+	Usage:    "List duplicate payments across all payments.",
+	Hidden:   true,
+	Action:   actionDecorator(listAllPaymentDuplicates),
+}
+
+func listAllPaymentDuplicates(ctx *cli.Context) error {
+	ctxc := getContext()
+	client, cleanUp := getClient(ctx)
+	defer cleanUp()
+
+	req := &lnrpc.ListAllPaymentDuplicatesRequest{}
+
+	resp, err := client.ListAllPaymentDuplicates(ctxc, req)
+	if err != nil {
+		return err
+	}
+
+	printRespJSON(resp)
+	return nil
+}
+
 var forwardingHistoryCommand = cli.Command{
 	Name:     "fwdinghistory",
 	Category: "Payments",

cmd/commands/main.go

@@ -482,6 +482,8 @@ func Main() {
 		ListChannelsCommand,
 		closedChannelsCommand,
 		listPaymentsCommand,
+		listPaymentDuplicatesCommand,
+		listAllPaymentDuplicatesCommand,
 		describeGraphCommand,
 		getNodeMetricsCommand,
 		getChanInfoCommand,

config_builder.go

@@ -51,6 +51,8 @@ import (
 	"github.com/lightningnetwork/lnd/macaroons"
 	"github.com/lightningnetwork/lnd/msgmux"
 	paymentsdb "github.com/lightningnetwork/lnd/payments/db"
+	paymentsmig1 "github.com/lightningnetwork/lnd/payments/db/migration1"
+	paymentsmig1sqlc "github.com/lightningnetwork/lnd/payments/db/migration1/sqlc"
 	"github.com/lightningnetwork/lnd/rpcperms"
 	"github.com/lightningnetwork/lnd/signal"
 	"github.com/lightningnetwork/lnd/sqldb"
@@ -76,6 +78,10 @@ const (
 	// graphMigration is the version number for the graph migration
 	// that migrates the KV graph to the native SQL schema.
 	graphMigration = 10
+
+	// paymentMigration is the version number for the payments migration
+	// that migrates KV payments to the native SQL schema.
+	paymentMigration = 12
 )
 
 // GrpcRegistrar is an interface that must be satisfied by an external subserver
@@ -1153,6 +1159,32 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 				return nil
 			}
 
+			paymentMig := func(tx *sqlc.Queries) error {
+				var err error
+				err = paymentsmig1.MigratePaymentsKVToSQL(
+					ctx,
+					dbs.ChanStateDB.Backend,
+					paymentsmig1sqlc.New(tx.GetTx()),
+					&paymentsmig1.SQLStoreConfig{
+						QueryCfg: queryCfg,
+					},
+				)
+				if err != nil {
+					return fmt.Errorf("failed to migrate "+
+						"payments to SQL: %w", err)
+				}
+
+				// Set the payments bucket tombstone to
+				// indicate that the migration has been
+				// completed.
+				d.logger.Debugf("Setting payments bucket " +
+					"tombstone")
+
+				return paymentsdb.SetPaymentsBucketTombstone(
+					dbs.ChanStateDB.Backend,
+				)
+			}
+
 			// Make sure we attach the custom migration function to
 			// the correct migration version.
 			for i := 0; i < len(migrations); i++ {
@@ -1162,11 +1194,17 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 					migrations[i].MigrationFn = invoiceMig
 
 					continue
+
 				case graphMigration:
 					migrations[i].MigrationFn = graphMig
 
 					continue
 
+				case paymentMigration:
+					migrations[i].MigrationFn = paymentMig
+
+					continue
+
 				default:
 				}
 
@@ -1228,6 +1266,23 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 
 			return nil, nil, err
 		}
+
+		// Create the payments DB.
+		//
+		// NOTE:  In the regular build, this will construct a kvdb
+		// backed payments backend. With the test_native_sql tag, it
+		// will build a SQL payments backend.
+		sqlPaymentsDB, err := d.getPaymentsStore(
+			baseDB, dbs.ChanStateDB.Backend,
+		)
+		if err != nil {
+			err = fmt.Errorf("unable to get payments store: %w",
+				err)
+
+			return nil, nil, err
+		}
+
+		dbs.PaymentsDB = sqlPaymentsDB
 	} else {
 		// Check if the invoice bucket tombstone is set. If it is, we
 		// need to return and ask the user switch back to using the
@@ -1248,6 +1303,27 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 			return nil, nil, err
 		}
 
+		// Check if the payments bucket tombstone is set. If it is, we
+		// need to return and ask the user switch back to using the
+		// native SQL store.
+		ripPayments, err := paymentsdb.GetPaymentsBucketTombstone(
+			dbs.ChanStateDB.Backend,
+		)
+		if err != nil {
+			err = fmt.Errorf("unable to check payments bucket "+
+				"tombstone: %w", err)
+			d.logger.Error(err)
+
+			return nil, nil, err
+		}
+		if ripPayments {
+			err = fmt.Errorf("payments bucket tombstoned, please " +
+				"switch back to native SQL")
+			d.logger.Error(err)
+
+			return nil, nil, err
+		}
+
 		dbs.InvoiceDB = dbs.ChanStateDB
 
 		graphStore, err = graphdb.NewKVStore(
@@ -1256,40 +1332,32 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 		if err != nil {
 			return nil, nil, err
 		}
-	}
 
-	dbs.GraphDB, err = graphdb.NewChannelGraph(graphStore, chanGraphOpts...)
-	if err != nil {
-		cleanUp()
+		// Create the payments DB.
+		kvPaymentsDB, err := paymentsdb.NewKVStore(
+			dbs.ChanStateDB,
+		)
+		if err != nil {
+			cleanUp()
 
-		err = fmt.Errorf("unable to open channel graph DB: %w", err)
-		d.logger.Error(err)
+			err = fmt.Errorf("unable to open payments DB: %w", err)
+			d.logger.Error(err)
 
-		return nil, nil, err
-	}
+			return nil, nil, err
+		}
 
-	// Mount the payments DB which is only KV for now.
-	//
-	// TODO(ziggie): Add support for SQL payments DB.
-	// Mount the payments DB for the KV store.
-	paymentsDBOptions := []paymentsdb.OptionModifier{
-		paymentsdb.WithKeepFailedPaymentAttempts(
-			cfg.KeepFailedPaymentAttempts,
-		),
+		dbs.PaymentsDB = kvPaymentsDB
 	}
-	kvPaymentsDB, err := paymentsdb.NewKVStore(
-		dbs.ChanStateDB,
-		paymentsDBOptions...,
-	)
+
+	dbs.GraphDB, err = graphdb.NewChannelGraph(graphStore, chanGraphOpts...)
 	if err != nil {
 		cleanUp()
 
-		err = fmt.Errorf("unable to open payments DB: %w", err)
+		err = fmt.Errorf("unable to open channel graph DB: %w", err)
 		d.logger.Error(err)
 
 		return nil, nil, err
 	}
-	dbs.PaymentsDB = kvPaymentsDB
 
 	// Wrap the watchtower client DB and make sure we clean up.
 	if cfg.WtClient.Active {

config_prod.go

@@ -6,6 +6,8 @@ import (
 	"context"
 
 	"github.com/lightningnetwork/lnd/kvdb"
+	paymentsdb "github.com/lightningnetwork/lnd/payments/db"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/lightningnetwork/lnd/sqldb/sqlc"
 )
 
@@ -24,3 +26,12 @@ func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context,
 
 	return nil, false
 }
+
+// getPaymentsStore returns a paymentsdb.DB backed by a paymentsdb.KVStore
+// implementation.
+func (d *DefaultDatabaseBuilder) getPaymentsStore(_ *sqldb.BaseDB,
+	kvBackend kvdb.Backend,
+	opts ...paymentsdb.OptionModifier) (paymentsdb.DB, error) {
+
+	return paymentsdb.NewKVStore(kvBackend, opts...)
+}

config_test_native_sql.go

@@ -4,8 +4,12 @@ package lnd
 
 import (
 	"context"
+	"database/sql"
 
 	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/lncfg"
+	paymentsdb "github.com/lightningnetwork/lnd/payments/db"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/lightningnetwork/lnd/sqldb/sqlc"
 )
 
@@ -25,3 +29,28 @@ func (d *DefaultDatabaseBuilder) getSQLMigration(_ context.Context,
 		return nil, false
 	}
 }
+
+// getPaymentsStore returns a paymentsdb.DB backed by a paymentsdb.SQLStore
+// implementation.
+func (d *DefaultDatabaseBuilder) getPaymentsStore(baseDB *sqldb.BaseDB,
+	kvBackend kvdb.Backend,
+	opts ...paymentsdb.OptionModifier) (paymentsdb.DB, error) {
+
+	paymentsExecutor := sqldb.NewTransactionExecutor(
+		baseDB, func(tx *sql.Tx) paymentsdb.SQLQueries {
+			return baseDB.WithTx(tx)
+		},
+	)
+
+	queryConfig := d.cfg.DB.Sqlite.QueryConfig
+	if d.cfg.DB.Backend == lncfg.PostgresBackend {
+		queryConfig = d.cfg.DB.Postgres.QueryConfig
+	}
+
+	return paymentsdb.NewSQLStore(
+		&paymentsdb.SQLStoreConfig{
+			QueryCfg: &queryConfig,
+		},
+		paymentsExecutor, opts...,
+	)
+}

docs/release-notes/release-notes-0.21.0.md

@@ -137,6 +137,32 @@
   code](https://github.com/lightningnetwork/lnd/pull/10338) to prevent the 
   need for maintenance as the sqlc code evolves. 
 
+* Payment Store SQL implementation and migration project:
+  * Introduce an [abstract payment 
+    store](https://github.com/lightningnetwork/lnd/pull/10153) interface and
+    refacotor the payment related LND code to make it more modular.
+  * Implement the SQL backend for the [payments 
+    database](https://github.com/lightningnetwork/lnd/pull/9147)
+  * Implement query methods (QueryPayments,FetchPayment) for the [payments db 
+    SQL Backend](https://github.com/lightningnetwork/lnd/pull/10287)
+  * Implement insert methods for the [payments db 
+    SQL Backend](https://github.com/lightningnetwork/lnd/pull/10291)
+  * Implement third(final) Part of SQL backend [payment
+  functions](https://github.com/lightningnetwork/lnd/pull/10368)
+  * Finalize SQL payments implementation [enabling unit and itests
+    for SQL backend](https://github.com/lightningnetwork/lnd/pull/10292)
+  * [Thread context through payment 
+    db functions Part 1](https://github.com/lightningnetwork/lnd/pull/10307)
+  * [Thread context through payment 
+    db functions Part 2](https://github.com/lightningnetwork/lnd/pull/10308)
+  * [Finalize SQL implementation for 
+    payments db](https://github.com/lightningnetwork/lnd/pull/10373)
+  * [Add the KV-to-SQL payment
+    migration](https://github.com/lightningnetwork/lnd/pull/10485) with 
+    comprehensive tests and build tag "test_native_sql" gated wiring into the 
+    payment flow.
+
+
 ## Code Health
 
 ## Tooling and Documentation
@@ -146,7 +172,9 @@
 * Boris Nagaev
 * Elle Mouton
 * Erick Cestari
+* Gijs van Dam
 * hieblmi
 * Mohamed Awnallah
 * Nishant Bansal
 * Pins
+* Ziggie

go.mod

@@ -141,7 +141,7 @@ require (
 	github.com/opencontainers/runc v1.1.14 // indirect
 	github.com/ory/dockertest/v3 v3.10.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect