diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index d2c6878a9d6a..49a2d4d841df 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -3568,6 +3568,7 @@ static void hci_le_conn_complete_evt(struct hci_dev *hdev, struct sk_buff *skb)
 {
 	struct hci_ev_le_conn_complete *ev = (void *) skb->data;
 	struct hci_conn *conn;
+	struct smp_irk *irk;
 
 	BT_DBG("%s status 0x%2.2x", hdev->name, ev->status);
 
@@ -3600,6 +3601,13 @@ static void hci_le_conn_complete_evt(struct hci_dev *hdev, struct sk_buff *skb)
 		}
 	}
 
+	/* Track the connection based on the Identity Address from now on */
+	irk = hci_get_irk(hdev, &ev->bdaddr, ev->bdaddr_type);
+	if (irk) {
+		bacpy(&conn->dst, &irk->bdaddr);
+		conn->dst_type = irk->addr_type;
+	}
+
 	if (ev->status) {
 		mgmt_connect_failed(hdev, &conn->dst, conn->type,
 				    conn->dst_type, ev->status);
diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c
index 8517d1f0984d..af29afed0cca 100644
--- a/net/bluetooth/smp.c
+++ b/net/bluetooth/smp.c
@@ -987,6 +987,10 @@ static int smp_cmd_ident_addr_info(struct l2cap_conn *conn,
 	hci_add_irk(conn->hcon->hdev, &smp->id_addr, smp->id_addr_type,
 		    smp->irk, &rpa);
 
+	/* Track the connection based on the Identity Address from now on */
+	bacpy(&hcon->dst, &smp->id_addr);
+	hcon->dst_type = smp->id_addr_type;
+
 	smp_distribute_keys(conn, 1);
 
 	return 0;