diff --git a/common/utxo.h b/common/utxo.h index 1652cc2c6e9e..226cdfc358b5 100644 --- a/common/utxo.h +++ b/common/utxo.h @@ -39,6 +39,9 @@ struct utxo { /* NULL if not spent yet, otherwise, the block the spending transaction is in */ const u32 *spendheight; + /* Block this utxo becomes unreserved, if applicable */ + const u32 *reserved_til; + /* The scriptPubkey if it is known */ u8 *scriptPubkey; diff --git a/common/wallet_tx.c b/common/wallet_tx.c index da9a1a171523..44af8382815c 100644 --- a/common/wallet_tx.c +++ b/common/wallet_tx.c @@ -91,11 +91,6 @@ struct command_result *param_utxos(struct command *cmd, tal_free(txids); tal_free(outnums); - if (!*utxos) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Could not decode all of the outpoints. The utxos" - " should be specified as an array of " - " 'txid:output_index'."); if (tal_count(*utxos) == 0) return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "No matching utxo was found from the wallet. " diff --git a/common/withdraw_tx.c b/common/withdraw_tx.c index b52aac517c84..f37caf8a19d5 100644 --- a/common/withdraw_tx.c +++ b/common/withdraw_tx.c @@ -12,6 +12,7 @@ struct bitcoin_tx *withdraw_tx(const tal_t *ctx, const struct chainparams *chainparams, + bool allow_rbf, const struct utxo **utxos, struct bitcoin_tx_output **outputs, const struct ext_key *bip32_base, @@ -19,10 +20,21 @@ struct bitcoin_tx *withdraw_tx(const tal_t *ctx, { struct bitcoin_tx *tx; int output_count; + /* + * BIP-125: Opt-in Full Replace-by-Fee Signaling + * https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki + * A transaction is considered to have opted in to + * allowing replacement of itself if any of its + * inputs have an nSequence number less than (0xffffffff - 1). + */ + /* A sequence of (0xffffffff - 1) signals to use locktime */ + u32 sequence = BITCOIN_TX_DEFAULT_SEQUENCE - 1; + if (allow_rbf) + sequence--; tx = tx_spending_utxos(ctx, chainparams, utxos, bip32_base, false, tal_count(outputs), nlocktime, - BITCOIN_TX_DEFAULT_SEQUENCE - 1); + sequence); output_count = bitcoin_tx_add_multi_outputs(tx, outputs); assert(output_count == tal_count(outputs)); diff --git a/common/withdraw_tx.h b/common/withdraw_tx.h index cf5d67af18c6..17b7458cd8e7 100644 --- a/common/withdraw_tx.h +++ b/common/withdraw_tx.h @@ -19,6 +19,7 @@ struct utxo; * * @ctx: context to tal from. * @chainparams: (in) the params for the created transaction. + * @allow_rbf: (in) bool to signal whether to flag the sequence as RBF'able * @utxos: (in/out) tal_arr of UTXO pointers to spend (permuted to match) * @outputs: (in) tal_arr of bitcoin_tx_output, scriptPubKeys with amount to send to. * @bip32_base: (in) bip32 base for key derivation, or NULL. @@ -26,6 +27,7 @@ struct utxo; */ struct bitcoin_tx *withdraw_tx(const tal_t *ctx, const struct chainparams *chainparams, + bool allow_rbf, const struct utxo **utxos, struct bitcoin_tx_output **outputs, const struct ext_key *bip32_base, diff --git a/contrib/pyln-client/pyln/client/lightning.py b/contrib/pyln-client/pyln/client/lightning.py index abba57f23fe5..ec961660481d 100644 --- a/contrib/pyln-client/pyln/client/lightning.py +++ b/contrib/pyln-client/pyln/client/lightning.py @@ -1107,7 +1107,7 @@ def txsend(self, txid): } return self.call("txsend", payload) - def reserveinputs(self, outputs, feerate=None, minconf=None, utxos=None): + def reserveinputs(self, outputs, feerate=None, minconf=None, utxos=None, expire_after=None, allow_rbf=None): """ Reserve UTXOs and return a psbt for a 'stub' transaction that spends the reserved UTXOs. @@ -1117,6 +1117,8 @@ def reserveinputs(self, outputs, feerate=None, minconf=None, utxos=None): "feerate": feerate, "minconf": minconf, "utxos": utxos, + "expire_after": expire_after, + "allow_rbf": allow_rbf, } return self.call("reserveinputs", payload) diff --git a/doc/lightning-reserveinputs.7 b/doc/lightning-reserveinputs.7 index 06db8c23a33f..8c7d808fac33 100644 --- a/doc/lightning-reserveinputs.7 +++ b/doc/lightning-reserveinputs.7 @@ -3,7 +3,7 @@ lightning-reserveinputs - Construct a transaction and reserve the UTXOs it spends .SH SYNOPSIS -\fBreserveinputs\fR \fIoutputs\fR [\fIfeerate\fR] [\fIminconf\fR] [\fIutxos\fR] +\fBreserveinputs\fR \fIoutputs\fR [\fIfeerate\fR] [\fIminconf\fR] [\fIutxos\fR] [\fIexpire_after\fR] [\fIallow_rbf\fR] .SH DESCRIPTION @@ -50,6 +50,15 @@ should have\. Default is 1\. \fIutxos\fR specifies the utxos to be used to fund the transaction, as an array of "txid:vout"\. These must be drawn from the node's available UTXO set\. + +\fIexpire_after\fR specifies the number of blocks after which the UTXOs reserved +by this command will be eligible for re-use\. Defaults to 144 blocks\. +Can be disabled by passing in 0\. + + +\fIallow_rbf\fR, if flagged on, will set the sequence for each input to permit +it to be RBF'd\. Defaults to true\. + .SH RETURN VALUE On success, an object with attributes \fIpsbt\fR and \fIfeerate_per_kw\fR will be diff --git a/doc/lightning-reserveinputs.7.md b/doc/lightning-reserveinputs.7.md index 0ee15a25a6b2..c2223db3f0db 100644 --- a/doc/lightning-reserveinputs.7.md +++ b/doc/lightning-reserveinputs.7.md @@ -4,7 +4,7 @@ lightning-reserveinputs -- Construct a transaction and reserve the UTXOs it spen SYNOPSIS -------- -**reserveinputs** *outputs* \[*feerate*\] \[*minconf*\] \[*utxos*\] +**reserveinputs** *outputs* \[*feerate*\] \[*minconf*\] \[*utxos*\] \[*expire_after*\] \[*allow_rbf*\] DESCRIPTION ----------- @@ -45,6 +45,13 @@ should have. Default is 1. *utxos* specifies the utxos to be used to fund the transaction, as an array of "txid:vout". These must be drawn from the node's available UTXO set. +*expire_after* specifies the number of blocks after which the UTXOs reserved +by this command will be eligible for re-use. Defaults to 144 blocks. +Can be disabled by passing in 0. + +*allow_rbf*, if flagged on, will set the sequence for each input to permit +it to be RBF'd. Defaults to true. + RETURN VALUE ------------ diff --git a/hsmd/hsmd.c b/hsmd/hsmd.c index b4e04705aae6..45f4013c24a3 100644 --- a/hsmd/hsmd.c +++ b/hsmd/hsmd.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include diff --git a/tests/test_wallet.py b/tests/test_wallet.py index 947d46700860..5c60cd0527ce 100644 --- a/tests/test_wallet.py +++ b/tests/test_wallet.py @@ -557,9 +557,86 @@ def test_reserveinputs(node_factory, bitcoind, chainparams): assert len([x for x in outputs if not x['reserved']]) == 0 assert len([x for x in outputs if x['reserved']]) == 12 - # FIXME: restart the node, nothing will remain reserved + # restart the node, things remain reserved l1.restart() - assert len(l1.rpc.listfunds()['outputs']) == 12 + outputs = l1.rpc.listfunds()['outputs'] + assert len([x for x in outputs if x['reserved']]) == 12 + + # move the blockchain forward 144 blocks (default reserved value) + l1.stop() + bitcoind.generate_block(144) + l1.start() + sync_blockheight(bitcoind, [l1]) + # everything should be back to unreserved now + outputs = l1.rpc.listfunds()['outputs'] + assert len([x for x in outputs if not x['reserved']]) == 12 + + # try setting the default reserved to 5 blocks + reserved = l1.rpc.reserveinputs([{addr: 'all'}], expire_after=5) + bitcoind.generate_block(5) + sync_blockheight(bitcoind, [l1]) + outputs = l1.rpc.listfunds()['outputs'] + assert len([x for x in outputs if not x['reserved']]) == 12 + + # try turning off the default reservation + reserved = l1.rpc.reserveinputs([{addr: 'all'}], expire_after=0) + bitcoind.generate_block(144) + sync_blockheight(bitcoind, [l1]) + outputs = l1.rpc.listfunds()['outputs'] + assert len([x for x in outputs if x['reserved']]) == 12 + + +def test_reserve_rbf(node_factory, bitcoind, chainparams): + """ + We now default everything to RBF. I'm not really sure how to test this + """ + amount = 1000000 + total_outs = 12 + coin_mvt_plugin = os.path.join(os.getcwd(), 'tests/plugins/coin_movements.py') + l1 = node_factory.get_node(options={'plugin': coin_mvt_plugin}, + feerates=(7500, 7500, 7500, 7500)) + l2 = node_factory.get_node() + addr = chainparams['example_addr'] + + # Add a medley of funds to withdraw later, bech32 + p2sh-p2wpkh + for i in range(total_outs // 2): + bitcoind.rpc.sendtoaddress(l1.rpc.newaddr()['bech32'], + amount / 10**8) + bitcoind.rpc.sendtoaddress(l1.rpc.newaddr('p2sh-segwit')['p2sh-segwit'], + amount / 10**8) + bitcoind.generate_block(1) + wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == total_outs) + + # Make a PSBT out of our inputs + reserved = l1.rpc.reserveinputs(outputs=[{addr: Millisatoshi(3 * amount * 1000)}]) + assert len([x for x in l1.rpc.listfunds()['outputs'] if x['reserved']]) == 4 + + signed_psbt = l1.rpc.signpsbt(reserved['psbt'])['signed_psbt'] + broadcast_tx = l1.rpc.sendpsbt(signed_psbt) + assert bitcoind.rpc.getmempoolinfo()['size'] == 1 + + # Try to do it again? + psbt = bitcoind.rpc.decodepsbt(reserved['psbt']) + utxos = [] + unreserve_utxos = [] + for vin in psbt['tx']['vin']: + utxos.append(vin['txid'] + ":" + str(vin['vout'])) + unreserve_utxos.append({'txid': vin['txid'], 'vout': vin['vout'], 'sequence': vin['sequence']}) + unreserve_psbt = bitcoind.rpc.createpsbt(unreserve_utxos, []) + unres = l1.rpc.unreserveinputs(unreserve_psbt) + + # let's not set the fee high enough, should explode + reserved = l1.rpc.reserveinputs([{addr: Millisatoshi(amount * 0.5 * 1000)}], feerate='7000perkw', utxos=utxos) + signed_psbt = l1.rpc.signpsbt(reserved['psbt'])['signed_psbt'] + with pytest.raises(RpcError, match=r'insufficient fee, rejecting replacement'): + l1.rpc.sendpsbt(signed_psbt) + + # now let's do it for reals + unres = l1.rpc.unreserveinputs(unreserve_psbt) + reserved = l1.rpc.reserveinputs([{addr: Millisatoshi(amount * 0.5 * 1000)}], feerate='10000perkw', utxos=utxos) + signed_psbt = l1.rpc.signpsbt(reserved['psbt'])['signed_psbt'] + l1.rpc.sendpsbt(signed_psbt) + assert bitcoind.rpc.getmempoolinfo()['size'] == 1 def test_sign_and_send_psbt(node_factory, bitcoind, chainparams): diff --git a/wallet/db.c b/wallet/db.c index c97d2e08b28d..1af228f1420e 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -617,6 +617,7 @@ static struct migration dbmigrations[] = { /* We track the counter for coin_moves, as a convenience for notification consumers */ {SQL("INSERT INTO vars (name, intval) VALUES ('coin_moves_count', 0);"), NULL}, {NULL, migrate_last_tx_to_psbt}, + {SQL("ALTER TABLE outputs ADD reserved_til INTEGER DEFAULT NULL;"), NULL}, }; /* Leak tracking. */ diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 469de169ea36..ff41497089e9 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -902,6 +902,10 @@ static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) struct node_id id; struct amount_sat fee_estimate, change_satoshis; const struct utxo **utxos; + u32 expires_at = 30; + + /* Set the chaintip height to 15 */ + ld->topology->tip->height = 15; CHECK(w); memset(&u, 0, sizeof(u)); @@ -966,6 +970,61 @@ static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) output_state_spent), "could not change output state ignoring oldstate"); + /* Check for 'reserved' state changes */ + utxos = wallet_select_coins(w, w, true, AMOUNT_SAT(1), 0, 21, + 0 /* no confirmations required */, + &fee_estimate, &change_satoshis); + CHECK(utxos && tal_count(utxos) == 1); + + u = *utxos[0]; + + /* Add expiration to reservation */ + CHECK_MSG(wallet_output_reservation_update(w, &u, expires_at), + "could not add high watermark for reserved output"); + + /* Usually we'd remove the reservation on free, but + * we want to do some things with it, so check that + * we can short-circuit this by calling persist */ + wallet_persist_utxo_reservation(w, utxos); + tal_free(utxos); + + /* Now should be unable to select 1 sat amount */ + CHECK_MSG(!wallet_select_coins(w, w, true, AMOUNT_SAT(1), 0, 21, + 0 /* no confirmations required */, + &fee_estimate, &change_satoshis), + "too many utxos available"); + + /* Get reserved utxos */ + utxos = (const struct utxo **)wallet_get_utxos(w, w, output_state_reserved); + CHECK(utxos && tal_count(utxos) == 1); + + /* Check that reserved_til is set */ + CHECK(*utxos[0]->reserved_til == expires_at); + + /* Expire the lease! */ + ld->topology->tip->height = expires_at; + + utxos = wallet_select_coins(w, w, true, AMOUNT_SAT(1), 0, 21, + 0 /* no confirmations required */, + &fee_estimate, &change_satoshis); + CHECK_MSG(utxos && tal_count(utxos) == 1, "utxo count is incorrect"); + + /* Check freeing them resets them to available */ + tal_free(utxos); + + /* There should be no reserved utxos */ + utxos = (const struct utxo **)wallet_get_utxos(w, w, output_state_reserved); + CHECK(tal_count(utxos) == 0); + + /* check that we can mark as spent */ + utxos = wallet_select_coins(w, w, true, AMOUNT_SAT(1), 0, 21, + 0 /* no confirmations required */, + &fee_estimate, &change_satoshis); + CHECK_MSG(utxos && tal_count(utxos) == 1, "utxo count is incorrect"); + wallet_confirm_utxos(w, utxos); + + tal_free(utxos); + /* Attempt to save an UTXO with close_info set, no commitment_point */ memset(&u.txid, 2, sizeof(u.txid)); u.amount = AMOUNT_SAT(5); @@ -980,9 +1039,9 @@ static bool test_wallet_outputs(struct lightningd *ld, const tal_t *ctx) utxos = wallet_select_coins(w, w, true, AMOUNT_SAT(5), 0, 21, 0 /* no confirmations required */, &fee_estimate, &change_satoshis); - CHECK(utxos && tal_count(utxos) == 2); + CHECK(utxos && tal_count(utxos) == 1); - u = *utxos[1]; + u = *utxos[0]; CHECK(u.close_info->channel_id == 42 && u.close_info->commitment_point == NULL && node_id_eq(&u.close_info->peer_id, &id)); @@ -1481,6 +1540,9 @@ int main(int argc, const char *argv[]) /* Only elements in ld we should access */ list_head_init(&ld->peers); + ld->topology = tal(ld, struct chain_topology); + ld->topology->tip = tal(ld, struct block); + node_id_from_hexstr("02a1633cafcc01ebfb6d78e39f687a1f0995c62fc95f51ead10a02ee0be551b5dc", 66, &ld->id); /* Accessed in peer destructor sanity check */ htlc_in_map_init(&ld->htlcs_in); diff --git a/wallet/wallet.c b/wallet/wallet.c index bba6704b18a9..1947110b8550 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -178,6 +178,7 @@ static struct utxo *wallet_stmt2output(const tal_t *ctx, struct db_stmt *stmt) utxo->spendheight = NULL; utxo->scriptPubkey = NULL; utxo->scriptSig = NULL; + utxo->reserved_til = NULL; if (!db_column_is_null(stmt, 9)) { blockheight = tal(utxo, u32); @@ -196,10 +197,24 @@ static struct utxo *wallet_stmt2output(const tal_t *ctx, struct db_stmt *stmt) tal_dup_arr(utxo, u8, db_column_blob(stmt, 11), db_column_bytes(stmt, 11), 0); } + if (!db_column_is_null(stmt, 12)) { + u32 reserved_til = db_column_int(stmt, 12); + utxo->reserved_til = tal_dup(utxo, u32, &reserved_til); + } return utxo; } +static bool utxo_reservation_expired(struct lightningd *ld, const struct utxo *utxo) +{ + return !utxo->reserved_til || *utxo->reserved_til <= ld->topology->tip->height; +} + +static bool utxo_reservation_clear(struct wallet *w, const struct utxo *utxo) +{ + return wallet_output_reservation_update(w, utxo, 0); +} + bool wallet_update_output_status(struct wallet *w, const struct bitcoin_txid *txid, const u32 outnum, enum output_status oldstatus, @@ -229,6 +244,79 @@ bool wallet_update_output_status(struct wallet *w, return changes > 0; } +static bool assert_utxo_status(struct wallet *w, + const struct bitcoin_txid *txid, + const u32 outnum, + const enum output_status state) +{ + struct db_stmt *stmt; + bool res; + + stmt = db_prepare_v2(w->db, SQL("SELECT COUNT(*) FROM outputs " + "WHERE status=? AND prev_out_tx=? " + "AND prev_out_index=?")); + db_bind_int(stmt, 0, output_status_in_db(state)); + db_bind_txid(stmt, 1, txid); + db_bind_int(stmt, 2, outnum); + + db_query_prepared(stmt); + if (db_step(stmt)) { + res = db_column_u64(stmt, 0) > 0; + } else + res = false; + + tal_free(stmt); + return res; +} + +/* Given a utxo, check that it is eligible to be marked as reserved. + * This used to be a simple state check, but since we added 'expireable' + * reservations, we can 're-reserve' inputs if the original reservation + * has expired, while clearing out the expiration info + */ +static bool wallet_reserve_output(struct wallet *w, struct utxo *u) +{ + if (u->status == output_state_reserved) { + if (!utxo_reservation_expired(w->ld, + cast_const(struct utxo *, u))) + return false; + + return utxo_reservation_clear(w, u); + } + + return wallet_update_output_status(w, &u->txid, u->outnum, + output_state_available, + output_state_reserved); +} + +bool wallet_output_reservation_update(struct wallet *w, + const struct utxo *utxo, + const u32 expires_at) +{ + struct db_stmt *stmt; + size_t changes; + + /* If not already reserved, say no */ + if (!assert_utxo_status(w, &utxo->txid, utxo->outnum, output_state_reserved)) + return false; + + stmt = db_prepare_v2(w->db, + SQL("UPDATE outputs" + " SET reserved_til = ?" + " WHERE prev_out_tx = ? AND prev_out_index = ?")); + if (expires_at == 0) + db_bind_null(stmt, 0); + else + db_bind_int(stmt, 0, expires_at); + + db_bind_txid(stmt, 1, &utxo->txid); + db_bind_int(stmt, 2, utxo->outnum); + db_exec_prepared_v2(stmt); + changes = db_count_changes(stmt); + tal_free(stmt); + return changes > 0; +} + struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, const enum output_status state) { struct utxo **results; @@ -249,6 +337,7 @@ struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, const enum ou ", confirmation_height" ", spend_height" ", scriptpubkey " + ", reserved_til " "FROM outputs")); } else { stmt = db_prepare_v2(w->db, SQL("SELECT" @@ -264,6 +353,7 @@ struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, const enum ou ", confirmation_height" ", spend_height" ", scriptpubkey " + ", reserved_til " "FROM outputs " "WHERE status= ? ")); db_bind_int(stmt, 0, output_status_in_db(state)); @@ -300,6 +390,7 @@ struct utxo **wallet_get_unconfirmed_closeinfo_utxos(const tal_t *ctx, ", confirmation_height" ", spend_height" ", scriptpubkey" + ", reserved_til" " FROM outputs" " WHERE channel_id IS NOT NULL AND " "confirmation_height IS NULL")); @@ -335,6 +426,7 @@ struct utxo *wallet_utxo_get(const tal_t *ctx, struct wallet *w, ", confirmation_height" ", spend_height" ", scriptpubkey" + ", reserved_til" " FROM outputs" " WHERE prev_out_tx = ?" " AND prev_out_index = ?")); @@ -355,10 +447,89 @@ struct utxo *wallet_utxo_get(const tal_t *ctx, struct wallet *w, return utxo; } +static struct utxo **expired_reserved_utxos_get(const tal_t *ctx, struct wallet *w) +{ + struct db_stmt *stmt; + struct utxo **results; + + stmt = db_prepare_v2(w->db, SQL("SELECT" + " prev_out_tx" + ", prev_out_index" + ", value" + ", type" + ", status" + ", keyindex" + ", channel_id" + ", peer_id" + ", commitment_point" + ", confirmation_height" + ", spend_height" + ", scriptpubkey " + ", reserved_til" + " FROM outputs" + " WHERE status = ?" + " AND reserved_til <= ?")); + db_bind_int(stmt, 0, output_status_in_db(output_state_reserved)); + db_bind_int(stmt, 1, w->ld->topology->tip->height); + + db_query_prepared(stmt); + + if (stmt->error) { + log_broken(w->log, "SQL error: %s", stmt->error); + return NULL; + } + + results = tal_arr(ctx, struct utxo *, 0); + while (db_step(stmt)) { + struct utxo *u = wallet_stmt2output(results, stmt); + tal_arr_expand(&results, u); + } + tal_free(stmt); + + return results; +} + +static struct utxo **spendable_utxos_get(const tal_t *ctx, struct wallet *w) +{ + struct utxo **reserved, **available; + size_t rcount; + + reserved = expired_reserved_utxos_get(ctx, w); + available = wallet_get_utxos(ctx, w, output_state_available); + + rcount = tal_count(reserved); + tal_resize(&reserved, rcount + tal_count(available)); + for (size_t i = 0; i < tal_count(available); i++) + reserved[rcount + i] = tal_steal(reserved, available[i]); + + tal_free(available); + return reserved; +} + bool wallet_unreserve_output(struct wallet *w, const struct bitcoin_txid *txid, const u32 outnum) { + struct utxo *utxo; + + utxo = wallet_utxo_get(NULL, w, txid, outnum); + if (utxo && utxo->reserved_til && utxo->status == output_state_reserved) { + if (w->ld->topology->tip->height <= *utxo->reserved_til) + log_info(w->log, "Unreserving utxo " + "%s:%u before expiration. blockheight is %u," + " would have unreserved at %u.", + type_to_string(tmpctx, struct bitcoin_txid, &utxo->txid), + utxo->outnum, + w->ld->topology->tip->height, + *utxo->reserved_til); + + if (!utxo_reservation_clear(w, utxo)) { + tal_free(utxo); + return false; + } + } + tal_free(utxo); + return wallet_update_output_status(w, txid, outnum, output_state_reserved, output_state_available); @@ -369,11 +540,9 @@ bool wallet_unreserve_output(struct wallet *w, */ static void unreserve_utxo(struct wallet *w, const struct utxo *unres) { - if (!wallet_update_output_status(w, &unres->txid, unres->outnum, - output_state_reserved, - output_state_available)) { + if (!wallet_unreserve_output(w, &unres->txid, + unres->outnum)) fatal("Unable to unreserve output"); - } } /** @@ -396,7 +565,7 @@ void wallet_confirm_utxos(struct wallet *w, const struct utxo **utxos) for (size_t i = 0; i < tal_count(utxos); i++) { if (!wallet_update_output_status( w, &utxos[i]->txid, utxos[i]->outnum, - output_state_reserved, output_state_spent)) { + output_state_any, output_state_spent)) { fatal("Unable to mark output as spent"); } } @@ -450,7 +619,7 @@ static const struct utxo **wallet_select(const tal_t *ctx, struct wallet *w, *fee_estimate = AMOUNT_SAT(0); *satoshi_in = AMOUNT_SAT(0); - available = wallet_get_utxos(ctx, w, output_state_available); + available = spendable_utxos_get(ctx, w); for (i = 0; i < tal_count(available); i++) { size_t input_weight; @@ -468,9 +637,7 @@ static const struct utxo **wallet_select(const tal_t *ctx, struct wallet *w, tal_arr_expand(&utxos, u); - if (!wallet_update_output_status( - w, &available[i]->txid, available[i]->outnum, - output_state_available, output_state_reserved)) + if (!wallet_reserve_output(w, u)) fatal("Unable to reserve output"); /* Input weight: txid + index + sequence */ @@ -551,25 +718,52 @@ const struct utxo **wallet_select_specific(const tal_t *ctx, struct wallet *w, const struct utxo **utxos = tal_arr(ctx, const struct utxo*, 0); tal_add_destructor2(utxos, destroy_utxos, w); - available = wallet_get_utxos(ctx, w, output_state_available); + available = wallet_get_utxos(ctx, w, output_state_any); for (i = 0; i < tal_count(txids); i++) { for (j = 0; j < tal_count(available); j++) { if (bitcoin_txid_eq(&available[j]->txid, txids[i]) && available[j]->outnum == *outnums[i]) { + switch (available[j]->status) { + case output_state_available: + if (!wallet_update_output_status( + w, &available[j]->txid, available[j]->outnum, + output_state_available, output_state_reserved)) + fatal("Unable to reserve output"); + break; + case output_state_reserved: + if (!utxo_reservation_expired(w->ld, available[j])) + log_info(w->log, "Selecting reserved utxo %s:%u", + type_to_string(tmpctx, struct bitcoin_txid, &available[j]->txid), + available[j]->outnum); + break; + case output_state_spent: + /* We'll let you attempt to re-spend utxos that we've already + * 'sent' now, since this is the only way to do it for RBF's*/ + if (available[j]->spendheight) { + log_info(w->log, "Attempting to spend an already spent UTXO %s:%u" + " (spent in block %u)", + type_to_string(tmpctx, struct bitcoin_txid, &available[j]->txid), + available[j]->outnum, + *available[j]->spendheight); + goto fail; + } + break; + default: + fatal("Unable to reserve output"); + } + struct utxo *u = tal_steal(utxos, available[j]); tal_arr_expand(&utxos, u); - - if (!wallet_update_output_status( - w, &available[j]->txid, available[j]->outnum, - output_state_available, output_state_reserved)) - fatal("Unable to reserve output"); } } } tal_free(available); return utxos; +fail: + tal_free(available); + return tal_free(utxos); } const struct utxo **wallet_select_all(const tal_t *ctx, struct wallet *w, @@ -3744,6 +3938,14 @@ static void process_utxo_result(struct bitcoind *bitcoind, enum output_status newstate = txout == NULL ? output_state_spent : output_state_available; + /* If the utxo is still unspent, don't reset it if the 'lease' + * on it hasn't expired yet */ + if (txout != NULL && utxos[0]->reserved_til) { + if (!utxo_reservation_expired(bitcoind->ld, utxos[0])) + goto next; + else + utxo_reservation_clear(bitcoind->ld->wallet, utxos[0]); + } log_unusual(bitcoind->ld->wallet->log, "wallet: reserved output %s/%u reset to %s", type_to_string(tmpctx, struct bitcoin_txid, &utxos[0]->txid), @@ -3753,6 +3955,7 @@ static void process_utxo_result(struct bitcoind *bitcoind, &utxos[0]->txid, utxos[0]->outnum, utxos[0]->status, newstate); +next: /* If we have more, resolve them too. */ tal_arr_remove(&utxos, 0); if (tal_count(utxos) != 0) { diff --git a/wallet/wallet.h b/wallet/wallet.h index 8213184ee820..784ab7901cda 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -1260,6 +1260,20 @@ void wallet_persist_utxo_reservation(struct wallet *w, const struct utxo **utxos bool wallet_unreserve_output(struct wallet *w, const struct bitcoin_txid *txid, const u32 outnum); +/** + * wallet_output_reservation_update - Add an expiration for a utxo reservation + * Will be returned to available upon expiration. + * + * Output must currently be in `reserved` state. Passing in 0 for `reserve_til` + * will expire the current reservation (if any) + * + * @w - wallet + * @utxo - output to update + * @reserve_til - block number to consider reservation expired */ +bool wallet_output_reservation_update(struct wallet *w, + const struct utxo *utxo, + const u32 reserve_til); + /** * Get a list of transactions that we track in the wallet. * diff --git a/wallet/walletrpc.c b/wallet/walletrpc.c index ae41a734ec99..a531c3fe38c8 100644 --- a/wallet/walletrpc.c +++ b/wallet/walletrpc.c @@ -160,6 +160,148 @@ static struct command_result *broadcast_and_wait(struct command *cmd, return command_still_pending(cmd); } +static struct command_result *build_outputs(struct command *cmd, + const char *buffer, + const jsmntok_t *outputstok, + struct unreleased_tx **utx, + size_t *out_len, + struct bitcoin_tx_output ***outputs) +{ + size_t i; + const jsmntok_t *t; + + if (outputstok->size == 0) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "Empty outputs"); + + *outputs = tal_arr(tmpctx, struct bitcoin_tx_output *, outputstok->size); + *out_len = 0; + (*utx)->wtx->all_funds = false; + (*utx)->wtx->amount = AMOUNT_SAT(0); + json_for_each_arr(i, t, outputstok) { + struct amount_sat *amount; + const u8 *destination; + enum address_parse_result res; + + /* output format: {destination: amount} */ + if (t->type != JSMN_OBJECT) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "The output format must be " + "{destination: amount}"); + + res = json_to_address_scriptpubkey(cmd, + chainparams, + buffer, &t[1], + &destination); + if (res == ADDRESS_PARSE_UNRECOGNIZED) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Could not parse destination address"); + else if (res == ADDRESS_PARSE_WRONG_NETWORK) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "Destination address is not on network %s", + chainparams->network_name); + + amount = tal(tmpctx, struct amount_sat); + if (!json_to_sat_or_all(buffer, &t[2], amount)) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "'%.*s' is a invalid satoshi amount", + t[2].end - t[2].start, buffer + t[2].start); + + *outputs[i] = new_tx_output(*outputs, *amount, + cast_const(u8 *, destination)); + *out_len += tal_count(destination); + + /* In fact, the maximum amount of bitcoin satoshi is 2.1e15. + * It can't be equal to/bigger than 2^64. + * On the hand, the maximum amount of litoshi is 8.4e15, + * which also can't overflow. */ + /* This means this destination need "all" satoshi we have. */ + if (amount_sat_eq(*amount, AMOUNT_SAT(-1ULL))) { + if (outputstok->size > 1) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "outputs[%zi]: this destination wants" + " all satoshi. The count of outputs" + " can't be more than 1. ", i); + (*utx)->wtx->all_funds = true; + /* `AMOUNT_SAT(-1ULL)` is the max permissible for `wallet_select_all`. */ + (*utx)->wtx->amount = *amount; + break; + } + + if (!amount_sat_add(&(*utx)->wtx->amount, (*utx)->wtx->amount, *amount)) + return command_fail(cmd, JSONRPC2_INVALID_PARAMS, + "outputs: The sum of first %zi outputs" + " overflow. ", i); + } + return NULL; +} + +static struct command_result *create_tx(struct command *cmd, + struct unreleased_tx **utx, + struct bitcoin_tx_output **outputs, + const struct utxo **chosen_utxos, + size_t out_len, + u32 *feerate_per_kw, + u32 minconf, + u32 locktime, + bool allow_rbf) +{ + struct command_result *result; + u32 maxheight; + struct pubkey *changekey; + + maxheight = minconf_to_maxheight(minconf, cmd->ld); + + if (chosen_utxos) + result = wtx_from_utxos((*utx)->wtx, *feerate_per_kw, + out_len, maxheight, + chosen_utxos); + else + result = wtx_select_utxos((*utx)->wtx, *feerate_per_kw, + out_len, maxheight); + + if (result) + return result; + + /* Because of the max limit of AMOUNT_SAT(-1ULL), + * `(*utx)->wtx->all_funds` won't change in `wtx_select_utxos()` */ + if ((*utx)->wtx->all_funds) + outputs[0]->amount = (*utx)->wtx->amount; + + /* Add the change as the last output */ + if (!amount_sat_eq((*utx)->wtx->change, AMOUNT_SAT(0))) { + struct bitcoin_tx_output *change_output; + + changekey = tal(tmpctx, struct pubkey); + if (!bip32_pubkey(cmd->ld->wallet->bip32_base, changekey, + (*utx)->wtx->change_key_index)) + return command_fail(cmd, LIGHTNINGD, "Change key generation failure"); + + change_output = new_tx_output(outputs, (*utx)->wtx->change, + scriptpubkey_p2wpkh(tmpctx, changekey)); + tal_arr_expand(&outputs, change_output); + } + + (*utx)->outputs = tal_steal(*utx, outputs); + (*utx)->tx = withdraw_tx(*utx, chainparams, allow_rbf, + (*utx)->wtx->utxos, + (*utx)->outputs, + cmd->ld->wallet->bip32_base, + locktime); + + bitcoin_txid((*utx)->tx, &(*utx)->txid); + + return NULL; +} + +static struct unreleased_tx *utx_new(const tal_t *ctx, + struct command *cmd) +{ + struct unreleased_tx *utx = tal(ctx, struct unreleased_tx); + utx->wtx = tal(utx, struct wallet_tx); + wtx_init(cmd, utx->wtx, AMOUNT_SAT(-1ULL)); + return utx; +} + /* Common code for withdraw and txprepare. * * Returns NULL on success, and fills in wtx, output and @@ -169,23 +311,19 @@ static struct command_result *json_prepare_tx(struct command *cmd, const char *buffer, const jsmntok_t *params, bool for_withdraw, + bool allow_rbf, struct unreleased_tx **utx, u32 *feerate) { - u32 *feerate_per_kw = NULL; struct command_result *result; - u32 *minconf, maxheight; - struct pubkey *changekey; + u32 *minconf, *feerate_per_kw = NULL, locktime = 0; struct bitcoin_tx_output **outputs; - const jsmntok_t *outputstok = NULL, *t; + const jsmntok_t *outputstok = NULL; const u8 *destination = NULL; - size_t out_len, i; + size_t out_len; const struct utxo **chosen_utxos = NULL; - u32 locktime = 0; - *utx = tal(cmd, struct unreleased_tx); - (*utx)->wtx = tal(*utx, struct wallet_tx); - wtx_init(cmd, (*utx)->wtx, AMOUNT_SAT(-1ULL)); + *utx = utx_new(cmd, cmd); if (!for_withdraw) { /* From v0.7.3, the new style for *txprepare* use array of outputs @@ -342,8 +480,8 @@ static struct command_result *json_prepare_tx(struct command *cmd, if (result) return result; } - - maxheight = minconf_to_maxheight(*minconf, cmd->ld); + if (feerate) + *feerate = *feerate_per_kw; /* *withdraw* command or old *txprepare* command. * Support only one output. */ @@ -353,117 +491,22 @@ static struct command_result *json_prepare_tx(struct command *cmd, destination); out_len = tal_count(outputs[0]->script); - goto create_tx; - } - - if (outputstok->size == 0) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "Empty outputs"); - - outputs = tal_arr(tmpctx, struct bitcoin_tx_output *, outputstok->size); - out_len = 0; - (*utx)->wtx->all_funds = false; - (*utx)->wtx->amount = AMOUNT_SAT(0); - json_for_each_arr(i, t, outputstok) { - struct amount_sat *amount; - const u8 *destination; - enum address_parse_result res; - - /* output format: {destination: amount} */ - if (t->type != JSMN_OBJECT) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "The output format must be " - "{destination: amount}"); - - res = json_to_address_scriptpubkey(cmd, - chainparams, - buffer, &t[1], - &destination); - if (res == ADDRESS_PARSE_UNRECOGNIZED) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Could not parse destination address"); - else if (res == ADDRESS_PARSE_WRONG_NETWORK) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "Destination address is not on network %s", - chainparams->network_name); - - amount = tal(tmpctx, struct amount_sat); - if (!json_to_sat_or_all(buffer, &t[2], amount)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "'%.*s' is a invalid satoshi amount", - t[2].end - t[2].start, buffer + t[2].start); - - outputs[i] = new_tx_output(outputs, *amount, - cast_const(u8 *, destination)); - out_len += tal_count(destination); - - /* In fact, the maximum amount of bitcoin satoshi is 2.1e15. - * It can't be equal to/bigger than 2^64. - * On the hand, the maximum amount of litoshi is 8.4e15, - * which also can't overflow. */ - /* This means this destination need "all" satoshi we have. */ - if (amount_sat_eq(*amount, AMOUNT_SAT(-1ULL))) { - if (outputstok->size > 1) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "outputs[%zi]: this destination wants" - " all satoshi. The count of outputs" - " can't be more than 1. ", i); - (*utx)->wtx->all_funds = true; - /* `AMOUNT_SAT(-1ULL)` is the max permissible for `wallet_select_all`. */ - (*utx)->wtx->amount = *amount; - break; - } - - if (!amount_sat_add(&(*utx)->wtx->amount, (*utx)->wtx->amount, *amount)) - return command_fail(cmd, JSONRPC2_INVALID_PARAMS, - "outputs: The sum of first %zi outputs" - " overflow. ", i); + return create_tx(cmd, utx, outputs, chosen_utxos, out_len, + feerate_per_kw, *minconf, + locktime, allow_rbf); } -create_tx: - if (chosen_utxos) - result = wtx_from_utxos((*utx)->wtx, *feerate_per_kw, - out_len, maxheight, - chosen_utxos); - else - result = wtx_select_utxos((*utx)->wtx, *feerate_per_kw, - out_len, maxheight); - + result = build_outputs(cmd, buffer, outputstok, utx, &out_len, + &outputs); if (result) return result; - /* Because of the max limit of AMOUNT_SAT(-1ULL), - * `(*utx)->wtx->all_funds` won't change in `wtx_select_utxos()` */ - if ((*utx)->wtx->all_funds) - outputs[0]->amount = (*utx)->wtx->amount; - - /* Add the change as the last output */ - if (!amount_sat_eq((*utx)->wtx->change, AMOUNT_SAT(0))) { - struct bitcoin_tx_output *change_output; - - changekey = tal(tmpctx, struct pubkey); - if (!bip32_pubkey(cmd->ld->wallet->bip32_base, changekey, - (*utx)->wtx->change_key_index)) - return command_fail(cmd, LIGHTNINGD, "Keys generation failure"); - - change_output = new_tx_output(outputs, (*utx)->wtx->change, - scriptpubkey_p2wpkh(tmpctx, changekey)); - tal_arr_expand(&outputs, change_output); - } - - (*utx)->outputs = tal_steal(*utx, outputs); - (*utx)->tx = withdraw_tx(*utx, chainparams, - (*utx)->wtx->utxos, - (*utx)->outputs, - cmd->ld->wallet->bip32_base, - locktime); - - bitcoin_txid((*utx)->tx, &(*utx)->txid); - - if (feerate) - *feerate = *feerate_per_kw; - return NULL; + return create_tx(cmd, utx, outputs, chosen_utxos, out_len, + feerate_per_kw, *minconf, + locktime, allow_rbf); } + static struct command_result *json_txprepare(struct command *cmd, const char *buffer, const jsmntok_t *obj UNNEEDED, @@ -473,7 +516,7 @@ static struct command_result *json_txprepare(struct command *cmd, struct command_result *res; struct json_stream *response; - res = json_prepare_tx(cmd, buffer, params, false, &utx, NULL); + res = json_prepare_tx(cmd, buffer, params, false, false, &utx, NULL); if (res) return res; @@ -600,7 +643,7 @@ static struct command_result *json_withdraw(struct command *cmd, struct unreleased_tx *utx; struct command_result *res; - res = json_prepare_tx(cmd, buffer, params, true, &utx, NULL); + res = json_prepare_tx(cmd, buffer, params, true, false, &utx, NULL); if (res) return res; @@ -881,8 +924,12 @@ static struct command_result *json_outputs(struct command *cmd, } else json_add_string(response, "status", "unconfirmed"); - json_add_bool(response, "reserved", - utxos[i]->status == output_state_reserved); + bool reserved = utxos[i]->status == output_state_reserved; + if (reserved && utxos[i]->reserved_til) { + reserved = + *utxos[i]->reserved_til > get_block_height(cmd->ld->topology); + } + json_add_bool(response, "reserved", reserved); json_object_end(response); } @@ -1194,11 +1241,38 @@ static struct command_result *json_reserveinputs(struct command *cmd, { struct command_result *res; struct json_stream *response; + const jsmntok_t *outputstok; struct unreleased_tx *utx; + const struct utxo **chosen_utxos; + struct bitcoin_tx_output **outputs; + u32 *feerate_per_kw, *minconf, *expire_after; + size_t out_len; + bool *allow_rbf; + + if (!param(cmd, buffer, params, + p_req("outputs", param_array, &outputstok), + p_opt("feerate", param_feerate, &feerate_per_kw), + p_opt_def("minconf", param_number, &minconf, 1), + p_opt("utxos", param_utxos, &chosen_utxos), + p_opt_def("expire_after", param_number, &expire_after, 144), + p_opt_def("allow_rbf", param_bool, &allow_rbf, true), + NULL)) + return command_param_failed(); - u32 feerate; + if (!feerate_per_kw) { + res = param_feerate_estimate(cmd, &feerate_per_kw, + FEERATE_OPENING); + if (res) + return res; + } + + utx = utx_new(cmd, cmd); - res = json_prepare_tx(cmd, buffer, params, false, &utx, &feerate); + res = build_outputs(cmd, buffer, outputstok, &utx, &out_len, &outputs); + if (res) + return res; + res = create_tx(cmd, &utx, outputs, chosen_utxos, out_len, + feerate_per_kw, *minconf, 0, *allow_rbf); if (res) return res; @@ -1206,10 +1280,17 @@ static struct command_result *json_reserveinputs(struct command *cmd, * around, so we remove the auto-cleanup that happens * when the utxo objects are free'd */ wallet_persist_utxo_reservation(cmd->ld->wallet, utx->wtx->utxos); + if (*expire_after > 0) { + u32 height = get_block_height(cmd->ld->topology) + *expire_after; + for (size_t i = 0; i < tal_count(utx->wtx->utxos); i++) + wallet_output_reservation_update(cmd->ld->wallet, + utx->wtx->utxos[i], + height); + } response = json_stream_success(cmd); json_add_psbt(response, "psbt", utx->tx->psbt); - json_add_u32(response, "feerate_per_kw", feerate); + json_add_u32(response, "feerate_per_kw", *feerate_per_kw); return command_success(cmd, response); } static const struct json_command reserveinputs_command = {