Skip to content

Commit 5785470

Browse files
AndrewWestbergAndrewWestberg
committed
fix(newm-chain): STUD-498 Expired transactions stuck in Live Utxo state
1 parent 6815700 commit 5785470

File tree

21 files changed

+184
-46
lines changed

21 files changed

+184
-46
lines changed

‎.github/workflows/ktlint.yml‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,14 @@ jobs:
3232
fi
3333
echo "SHA256 verification passed"
3434
35+
#Temporarily disable gpg checks due to keybase not returning the key
3536
# Import ktlint PGP key and verify signature using official method
36-
curl -sS https://keybase.io/ktlint/pgp_keys.asc | gpg --import && gpg --verify ktlint.asc
37-
if [ $? -ne 0 ]; then
38-
echo "PGP signature verification failed!"
39-
exit 1
40-
fi
41-
echo "PGP signature verification passed"
37+
# curl -sS https://keybase.io/ktlint/pgp_keys.asc | gpg --import && gpg --verify ktlint.asc
38+
# if [ $? -ne 0 ]; then
39+
# echo "PGP signature verification failed!"
40+
# exit 1
41+
# fi
42+
# echo "PGP signature verification passed"
4243

4344
# Install ktlint
4445
chmod a+x ktlint

‎build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ plugins {
2424

2525
allprojects {
2626
group = "io.newm.server"
27-
version = "0.10.0-SNAPSHOT"
27+
version = "0.11.0-SNAPSHOT"
2828
}
2929

3030
subprojects {

‎buildSrc/src/main/kotlin/Versions.kt‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@ object Versions {
55
const val APACHE_CURATORS = "5.9.0"
66
const val APACHE_TIKA = "3.2.2"
77
const val ARWEAVE4S = "0.21.0"
8-
const val AWS = "2.32.30"
8+
const val AWS = "2.33.9"
99
const val BOUNCY_CASTLE = "1.70"
1010
const val CAFFEINE = "3.2.2"
1111
const val CBOR = "0.4.1-NEWM"
1212
const val CLOUDINARY = "1.39.0"
1313
const val COROUTINES = "1.10.2"
1414
const val EXPOSED = "0.61.0"
15-
const val FLYWAYDB = "11.11.2"
16-
const val GOOGLE_TRUTH = "1.4.4"
15+
const val FLYWAYDB = "11.12.0"
16+
const val GOOGLE_TRUTH = "1.4.5"
1717
const val GRPC = "1.75.0"
1818
const val GRPC_KOTLIN = "1.4.3"
1919
const val HIKARICP = "7.0.2"
@@ -23,30 +23,30 @@ object Versions {
2323
const val JUNIT = "5.13.4"
2424
const val J_AUDIO_TAGGER = "3.0.1"
2525
const val KOIN = "4.1.0-Beta8"
26-
const val KOIN_TEST = "4.1.0"
26+
const val KOIN_TEST = "4.1.1"
2727
const val KOGMIOS = "2.5.1"
2828
const val KOTLINX_SERIALIZATION = "1.9.0"
2929
const val KOTLIN_LOGGING = "7.0.13"
30-
const val KOTLIN_PLUGIN = "2.2.10"
30+
const val KOTLIN_PLUGIN = "2.2.20"
3131
const val KTLINT = "1.7.1"
3232
const val KTLINT_PLUGIN = "12.1.1"
33-
const val KTOR = "3.2.3"
33+
const val KTOR = "3.3.0"
3434
const val KTOR_FLYWAY = "3.0.0"
3535
const val LOGBACK = "1.5.18"
3636
const val MAVEN_PUBLISH = "0.34.0"
3737
const val MOCKK = "1.14.5"
3838
const val POSTGRESQL = "42.7.7"
39-
const val PROTOBUF = "4.32.0"
39+
const val PROTOBUF = "4.32.1"
4040
const val PROTOBUF_PLUGIN = "0.9.5"
4141
const val QR_CODE_KOTLIN = "4.5.0"
4242
const val QUARTZ = "2.5.0"
4343
const val SCALA_JAVA8_COMPAT = "1.0.2"
44-
const val SENTRY = "8.20.0"
44+
const val SENTRY = "8.21.1"
4545
const val SHADOW_PLUGIN = "8.1.1"
4646
const val SPRING_SECURITY = "6.5.3"
47-
const val SSL_KICKSTART = "9.1.0"
47+
const val SSL_KICKSTART = "9.2.1"
4848
const val SWAGGER = "1.0.57"
4949
const val TEST_CONTAINERS = "1.21.3"
50-
const val TYPESAFE = "1.4.4"
50+
const val TYPESAFE = "1.4.5"
5151
const val VERSIONS_PLUGIN = "0.52.0"
5252
}

‎gradle.properties‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
11
kotlin.code.style=official
2+
3+
org.gradle.parallel=true
4+
org.gradle.caching=true
5+
org.gradle.configureondemand=true
6+
org.gradle.daemon=true
7+
org.gradle.unsafe.configuration-cache=true
8+
29
org.gradle.jvmargs=-Dfile.encoding=UTF-8 \
310
--add-opens jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED \
411
--add-opens jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
@@ -8,3 +15,8 @@ org.gradle.jvmargs=-Dfile.encoding=UTF-8 \
815
--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED \
916
--add-opens jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
1017
--add-opens jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
18+
19+
kotlin.incremental=true
20+
kotlin.incremental.js=true
21+
kotlin.caching.enabled=true
22+
kotlin.parallel.tasks.in.project=true

‎newm-chain-db/src/main/kotlin/io/newm/chain/database/repository/ChainRepositoryImpl.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package io.newm.chain.database.repository
22

33
import com.github.benmanes.caffeine.cache.Caffeine
4-
import io.newm.chain.config.Config
4+
import io.newm.chain.util.config.Config
55
import io.newm.chain.database.entity.ChainBlock
66
import io.newm.chain.database.entity.MonitoredAddressChain
77
import io.newm.chain.database.table.ChainTable

‎newm-chain-db/src/main/kotlin/io/newm/chain/database/repository/LedgerRepositoryImpl.kt‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import com.google.iot.cbor.CborByteString
66
import com.google.iot.cbor.CborInteger
77
import com.google.iot.cbor.CborMap
88
import com.google.iot.cbor.CborReader
9-
import io.newm.chain.config.Config
9+
import io.newm.chain.util.config.Config
1010
import io.newm.chain.database.entity.LedgerAsset
1111
import io.newm.chain.database.entity.LedgerAssetMetadata
1212
import io.newm.chain.database.entity.LedgerUtxoHistory
@@ -42,10 +42,12 @@ import io.newm.chain.util.Constants.UTXO_SCRIPT_REF_INDEX
4242
import io.newm.chain.util.elementToByteArray
4343
import io.newm.chain.util.elementToInt
4444
import io.newm.chain.util.extractCredentials
45+
import io.newm.chain.util.getInstantAtSlot
4546
import io.newm.chain.util.hexToByteArray
4647
import io.newm.chain.util.toHexString
4748
import java.math.BigInteger
4849
import java.time.Duration
50+
import java.time.Instant
4951
import kotlin.math.max
5052
import kotlinx.coroutines.runBlocking
5153
import kotlinx.coroutines.sync.Mutex
@@ -89,6 +91,9 @@ class LedgerRepositoryImpl : LedgerRepository {
8991
*/
9092
private val liveUtxoMap = mutableMapOf<String, Set<Utxo>>()
9193

94+
// Track validity end slot (TTL) for pending transactions that produced liveUtxos
95+
private val pendingTxValidityEndSlots = mutableMapOf<String, Long>()
96+
9297
override fun queryUtxos(address: String): Set<Utxo> =
9398
transaction {
9499
warnLongQueriesDuration = 1000L
@@ -350,6 +355,7 @@ class LedgerRepositoryImpl : LedgerRepository {
350355
chainUtxos: MutableSet<Utxo>
351356
): Set<Utxo> {
352357
utxoMutex.withLock {
358+
cleanupExpiredPendingTransactions()
353359
return chainUtxos
354360
.apply {
355361
// add in any liveUtxos from pending transactions
@@ -386,9 +392,14 @@ class LedgerRepositoryImpl : LedgerRepository {
386392
cborByteArray: ByteArray
387393
) {
388394
utxoMutex.withLock {
395+
cleanupExpiredPendingTransactions()
389396
val tx = CborReader.createFromByteArray(cborByteArray).readDataItem() as CborArray
390397
val txBody = tx.elementAt(0) as CborMap
391398
val witnessSet = tx.elementAt(1) as CborMap
399+
// Record TTL (validityEndSlot) if present so we can expire pending UTxOs later
400+
(txBody[CborInteger.create(3)] as? CborInteger)?.longValue()?.let { ttl ->
401+
pendingTxValidityEndSlots[transactionId] = ttl
402+
}
392403
val utxosInArray = txBody[TX_SPENT_UTXOS_INDEX] as CborArray
393404
utxosInArray.forEach { utxo ->
394405
var hash = ""
@@ -554,6 +565,37 @@ class LedgerRepositoryImpl : LedgerRepository {
554565
}
555566
)
556567
}
568+
// After adding new pending utxos, perform another cleanup in case they are already expired
569+
cleanupExpiredPendingTransactions()
570+
}
571+
}
572+
573+
// Purge any pending transactions whose TTL has passed
574+
private fun cleanupExpiredPendingTransactions() {
575+
if (pendingTxValidityEndSlots.isEmpty()) return
576+
val now = Instant.now()
577+
val expiredTxIds = pendingTxValidityEndSlots.filter { (_, slot) -> now.isAfter(getInstantAtSlot(slot)) }.keys
578+
if (expiredTxIds.isEmpty()) return
579+
expiredTxIds.forEach { pendingTxValidityEndSlots.remove(it) }
580+
// Unmark spent UTXOs from expired transactions
581+
if (spentUtxoSet.isNotEmpty()) {
582+
spentUtxoSet.removeIf { it.transactionSpent in expiredTxIds }
583+
}
584+
// Remove created UTXOs from expired transactions
585+
if (liveUtxoMap.isNotEmpty()) {
586+
val removeAddresses = mutableListOf<String>()
587+
liveUtxoMap.forEach { (address, utxos) ->
588+
val newSet = utxos.filterNot { it.hash in expiredTxIds }.toSet()
589+
if (newSet.isEmpty()) {
590+
if (utxos.isNotEmpty()) removeAddresses.add(address)
591+
} else if (newSet.size != utxos.size) {
592+
liveUtxoMap[address] = newSet
593+
}
594+
}
595+
removeAddresses.forEach { liveUtxoMap.remove(it) }
596+
}
597+
if (log.isDebugEnabled) {
598+
log.debug("cleanupExpiredPendingTransactions: expiredTxIds=$expiredTxIds")
557599
}
558600
}
559601

‎newm-chain-db/src/main/kotlin/io/newm/chain/util/DbKtx.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package io.newm.chain.util
22

3-
import io.newm.chain.config.Config
3+
import io.newm.chain.util.config.Config
44
import io.newm.chain.database.entity.ChainBlock
55
import io.newm.chain.database.entity.LedgerAsset
66
import io.newm.chain.database.entity.LedgerAssetMetadata

‎newm-chain-util/build.gradle.kts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ dependencies {
2323

2424
implementation(Dependencies.LogBack.CLASSIC)
2525

26+
implementation(Dependencies.Newm.KOGMIOS)
27+
2628
implementation(Dependencies.Cbor.CBOR)
2729
implementation(Dependencies.ApacheCommonsCodec.ALL)
2830
implementation(Dependencies.BouncyCastle.BCPROV)

‎newm-chain/src/main/kotlin/io/newm/chain/cardano/ByronGenesis.kt‎ renamed to ‎newm-chain-util/src/main/kotlin/io/newm/chain/util/ByronGenesis.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package io.newm.chain.cardano
1+
package io.newm.chain.util
22

33
data class ByronGenesis(
44
val startTime: Long,

‎newm-chain-util/src/main/kotlin/io/newm/chain/util/Constants.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,10 @@ object Constants {
7474

7575
val TX_SPENT_UTXOS_INDEX: CborInteger = CborInteger.create(0)
7676
val TX_DEST_UTXOS_INDEX: CborInteger = CborInteger.create(1) // destination addresses are at index 1
77+
val TX_FEE_INDEX: CborInteger = CborInteger.create(2)
78+
val TX_TTL_INDEX: CborInteger = CborInteger.create(3)
7779
val TX_CERTS_INDEX: CborInteger = CborInteger.create(4)
80+
val TX_VALIDITY_INTERVAL_START = CborInteger.create(8)
7881
val TX_MINTS_INDEX: CborInteger = CborInteger.create(9)
7982
val TX_COLLAT_UTXOS_INDEX: CborInteger = CborInteger.create(13)
8083
val TX_WITNESS_DATUM_INDEX: CborInteger = CborInteger.create(4)

0 commit comments

Comments
 (0)