Askrene: fees get its probability cost

plugins/askrene/child/mcf.c

@@ -291,6 +291,26 @@ struct pay_parameters {
 	// how much we pay
 	struct amount_msat amount;
 
+	/* Introducing flow_fee_adjustment (here 1/g)
+	 *
+	 * MCF preserves flow at intermediate hops, therefore fees do not
+	 * contribute to flows or flows costs.  We can exceed capacity limits
+	 * once fees are added and/or hit very high probability costs
+	 * triggered by them.  To mitigate this we scale down the min/max limits
+	 * by this factor.  This works because multiplying the flow by g produces the same cost
+	 * than multiplying the min/max bounds by 1/g.
+	 *
+	 * We want a new cost function, assume fees = 1% of total amount
+	 * 	C'(x) = C(x + fees) where x+fees = x*(1+0.01)= x*g
+	 *
+	 * C'(x) = C(x*g) =
+	 * 	case x*g <= a, same as x <= a/g: 0
+	 * 	case x*g >= b, same as x >= b/g: infinity
+	 * 	case a<=x*g<b, same as a/g<=x<b/g: -log(1-(x*g-a)/(b-a)) =
+	 * -log(1-(x-a/g)/(b/g-a/g))
+	 * */
+	double flow_fee_adjustment;
+
 	/* base unit for computation, ie. accuracy */
 	struct amount_msat accuracy;
 
@@ -360,7 +380,8 @@ static double pickhardt_richter_probability(struct amount_msat low,
 /* Split a directed channel into parts with linear cost function. */
 static void linearize_channel(const struct pay_parameters *params,
 			      const struct gossmap_chan *c, const int dir,
-			      s64 *capacity, double *cost)
+			      s64 *capacity, double *cost,
+			      double flow_fee_adjustment)
 {
 	struct amount_msat mincap, maxcap;
 
@@ -371,6 +392,18 @@ static void linearize_channel(const struct pay_parameters *params,
 	if (amount_msat_greater(mincap, maxcap))
 		mincap = maxcap;
 
+	/* Allow headroom for fees */
+	if (!amount_msat_scale(&mincap, mincap, flow_fee_adjustment)) {
+		child_log(tmpctx, LOG_UNUSUAL,
+			  "%s: Couldn't scale down mincap=%s by %lf", __func__,
+			  fmt_amount_msat(tmpctx, mincap), flow_fee_adjustment);
+	}
+	if (!amount_msat_scale(&maxcap, maxcap, flow_fee_adjustment)) {
+		child_log(tmpctx, LOG_UNUSUAL,
+			  "%s: Couldn't scale down maxcap=%s by %lf", __func__,
+			  fmt_amount_msat(tmpctx, maxcap), flow_fee_adjustment);
+	}
+
 	u64 a = amount_msat_ratio_floor(mincap, params->accuracy),
 	    b = 1 + amount_msat_ratio_floor(maxcap, params->accuracy);
 
@@ -659,7 +692,8 @@ static void init_linear_network(const tal_t *ctx,
 
 			// split this channel direction to obtain the arcs
 			// that are outgoing to `node`
-			linearize_channel(params, c, half, capacity, prob_cost);
+			linearize_channel(params, c, half, capacity, prob_cost,
+					  params->flow_fee_adjustment);
 
 			/* linear fee_cost per unit of flow */
 			const double fee_cost = linear_fee_cost(
@@ -1024,6 +1058,7 @@ static struct flow **minflow(const tal_t *ctx,
 			     const struct gossmap_node *source,
 			     const struct gossmap_node *target,
 			     struct amount_msat amount,
+			     double flow_fee_adjustment,
 			     double mu,
 			     double delay_dial,
 			     double per_hop_dial,
@@ -1040,6 +1075,7 @@ static struct flow **minflow(const tal_t *ctx,
 	params->source = source;
 	params->target = target;
 	params->amount = amount;
+	params->flow_fee_adjustment = flow_fee_adjustment;
 	/* -> We reduce the granularity of the flow by limiting the subdivision
 	 * of the payment amount into 1000 units of flow. That reduces the
 	 * computational burden for algorithms that depend on it, eg. "capacity
@@ -1250,6 +1286,7 @@ static void init_linear_network_single_path(
  * @source: the source to start from
  * @target: the target to pay
  * @amount: the amount we want to reach @target
+ * @maxfee: fee constraint
  * @mu: 0 = corresponds to only probabilities, 1 corresponds to only fee.
  * @delay_dial: same as mu but for delays.
  * @prob_dial: same as mu but for probabilities.
@@ -1260,6 +1297,7 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query
 				      const struct gossmap_node *source,
 				      const struct gossmap_node *target,
 				      struct amount_msat amount,
+				      double flow_fee_adjustment UNUSED,
 				      double mu,
 				      double delay_dial,
 				      double per_hop_dial,
@@ -1464,6 +1502,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq,
 				      double,
 				      double,
 				      double,
+				      double,
 				      double))
 {
 	/* Almost 1 to limit the dials' iterations. */
@@ -1472,6 +1511,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq,
 	const char *error_message;
 	struct amount_msat amount_to_deliver = amount;
 	struct amount_msat feebudget = maxfee;
+	struct amount_msat amount_and_fees;
 
 	/* These are the cost dials for the feature costs we combine.
 	 * Each one must be a number between 0 and 1, and the sum of all must be
@@ -1481,6 +1521,29 @@ linear_routes(const tal_t *ctx, struct route_query *rq,
 	double per_hop_dial = 0.001;
 	double prob_dial = 0.988;
 
+	if (!amount_msat_add(&amount_and_fees, amount, maxfee)) {
+		child_log(tmpctx, LOG_BROKEN,
+			  "%s: Couldn't compute amount (%s) + maxfee (%s)",
+			  __func__, fmt_amount_msat(tmpctx, amount),
+			  fmt_amount_msat(tmpctx, maxfee));
+		/* fallback to only the amount */
+		amount_and_fees = amount;
+	}
+
+	double flow_fee_adjustment = amount_msat_ratio(amount, amount_and_fees);
+
+	/* The user might set maxfee to a very high value which reduces our
+	 * flow_fee_adjustment (1/g) limiting channel's effective capacity.
+	 * We prevent that by adding a lower cap to flow_fee_adjustment
+	 * at 0.99, which if roughly equivalent to a ceil on the user's fee
+	 * input at 1%.
+	 * For small payments the actual fees could be more than 100%, but still
+	 * being small payments the fee adjustment is not important, so 1.0 or
+	 * 0.99 flow_fee_adjustment doesn't make a difference. */
+	if (flow_fee_adjustment < 0.99)
+		flow_fee_adjustment = 0.99;
+	assert(flow_fee_adjustment <= 1.0);
+
 	/* Counters to keep a record of how many times the dials have been
 	 * updated. */
 	int count_mu_updates = 0, count_delay_updates = 0,
@@ -1557,6 +1620,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq,
 						     srcnode,
 						     dstnode,
 						     amount_to_deliver,
+						     flow_fee_adjustment,
 						     mu,
 						     delay_dial,
 						     per_hop_dial,
@@ -1568,6 +1632,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq,
 					   srcnode,
 					   dstnode,
 					   amount_to_deliver,
+					   flow_fee_adjustment,
 					   mu,
 					   delay_dial,
 					   per_hop_dial,
@@ -1576,6 +1641,17 @@ linear_routes(const tal_t *ctx, struct route_query *rq,
 		}
 
 		if (!new_flows) {
+			/* Could not find a solution, is this due to a very low
+			 * flow_fee_adjustment? */
+			if (flow_fee_adjustment < near_one) {
+				flow_fee_adjustment = 1;
+				child_log(tmpctx, LOG_INFORM,
+					  "Could not find routes with enough "
+					  "capacity, retrying without a "
+					  "flow_fee_adjustment");
+				continue;
+			}
+
 			error_message = explain_failure(
 				ctx, rq, srcnode, dstnode, amount_to_deliver, ecode);
 			goto fail;

tests/test_askrene.py

@@ -1759,10 +1759,10 @@ def test_real_biases(node_factory, bitcoind, executor):
     # CI, it's slow.
     if SLOW_MACHINE:
         limit = 25
-        expected = ({1: 5, 2: 6, 4: 13, 8: 16, 16: 22, 32: 24, 64: 24, 100: 24}, 0)
+        expected = ({1: 5, 2: 7, 4: 13, 8: 16, 16: 22, 32: 24, 64: 24, 100: 24}, 0)
     else:
         limit = 100
-        expected = ({1: 23, 2: 32, 4: 52, 8: 64, 16: 89, 32: 95, 64: 95, 100: 95}, 0)
+        expected = ({1: 23, 2: 34, 4: 52, 8: 64, 16: 89, 32: 95, 64: 95, 100: 95}, 0)
 
     l1.rpc.askrene_create_layer('biases')
     num_changed = {}
@@ -3327,3 +3327,120 @@ def test_bias_aging(node_factory):
     }
     expect["node_biases"] = []
     assert l1.rpc.askrene_listlayers("test_layers") == {"layers": [expect]}
+
+
+def test_flows_account_for_fees(node_factory):
+    """A probability function that neglects the fees together with the flows
+    would fail this test producing a far from optimal flow partition."""
+    known_min = 50_000_000
+    known_max = 50_050_000
+    pay_amount = 60_000_000
+    chan02_capacity = 1000_000
+    btc_capacity = 100_000_000
+    # 0->1->3 is cheap in probability up to 50k sats, but 50k plus fees nearly
+    # touches the known maximum there so the optimal solution is to send a value
+    # x through 0->1->3 such that x+fees == 50k and send the rest through
+    # 0->2->3 that even with unkown liquidity it is a huge channel.
+    gsfile, nodemap = generate_gossip_store(
+        [
+            GenChannel(0, 1, capacity_sats=1000_000),
+            GenChannel(
+                1, 3, forward=GenChannel.Half(propfee=1000), capacity_sats=btc_capacity
+            ),
+            GenChannel(0, 2, capacity_sats=chan02_capacity),
+            GenChannel(
+                2, 3, forward=GenChannel.Half(propfee=1000), capacity_sats=btc_capacity
+            ),
+        ]
+    )
+    l1 = node_factory.get_node(gossip_store_file=gsfile.name)
+
+    chan01 = only_one(
+        [
+            c
+            for c in l1.rpc.listchannels(source=nodemap[0])["channels"]
+            if c["destination"] == nodemap[1]
+        ]
+    )
+    scidd01 = f"{chan01['short_channel_id']}/{chan01['direction']}"
+    chan02 = only_one(
+        [
+            c
+            for c in l1.rpc.listchannels(source=nodemap[0])["channels"]
+            if c["destination"] == nodemap[2]
+        ]
+    )
+    scidd02 = f"{chan02['short_channel_id']}/{chan02['direction']}"
+    l1.rpc.askrene_create_layer("known_cap")
+    l1.rpc.askrene_inform_channel(
+        layer="known_cap",
+        short_channel_id_dir=scidd01,
+        amount_msat=known_min,
+        inform="unconstrained",
+    )
+    l1.rpc.askrene_inform_channel(
+        layer="known_cap",
+        short_channel_id_dir=scidd01,
+        amount_msat=known_max + 1,
+        inform="constrained",
+    )
+
+    # A generous maxfee must not prevent us from using the known capacity.
+    for maxfee in (
+        pay_amount // 900,
+        pay_amount // 500,
+        pay_amount // 200,
+        pay_amount // 100,
+        pay_amount // 50,
+        pay_amount // 10,
+        pay_amount,
+    ):
+        routes = l1.rpc.getroutes(
+            source=nodemap[0],
+            destination=nodemap[3],
+            amount_msat=pay_amount,
+            layers=["known_cap"],
+            maxfee_msat=maxfee,
+            final_cltv=99,
+        )["routes"]
+        assert sum(r["amount_msat"] for r in routes) == pay_amount
+
+        # We use the known channel almost to its full capacity, fees included,
+        # but never beyond it.
+        amounts01 = [
+            h["amount_out_msat"]
+            for r in routes
+            for h in r["path"]
+            if h["short_channel_id_dir"] == scidd01
+        ]
+        assert len(amounts01) == 1
+        amount01 = amounts01[0]
+        amounts02 = [
+            h["amount_out_msat"]
+            for r in routes
+            for h in r["path"]
+            if h["short_channel_id_dir"] == scidd02
+        ]
+        assert len(amounts02) == 1
+        amount02 = amounts02[0]
+
+        # total at 0->1 and 0->2 must match the final destination amount plus
+        # fees that are the same over each route.
+        assert amount01 + amount02 == 60_060_000
+
+        # over 0->1->3 we use almost all of its known_min, that is a necessary
+        # condition for optimality
+        assert known_min * 0.98 <= amount01
+
+        # whatever that is, it must not exceed the known_max
+        assert amount01 <= known_max
+
+        prob_fail01 = (
+            (amount01 - known_min) / (known_max - known_min)
+            if amount01 >= known_min
+            else 0.0
+        )
+        prob_fail02 = (amount02) / (chan02_capacity * 1000)
+
+        # the optimal solution is when prob_fail01 is near prob_fail02
+        assert abs(prob_fail01 - prob_fail02) < 0.02