Skip to content

Commit 3a1a5d0

Browse files
halibobo1205halibobo1205
authored
feat(db): update to rocksdb 9 for arm (#6440)
* feat(db): update to rocks9 for arm * feat(rocksdb): release option when db is created
1 parent 02f81c6 commit 3a1a5d0

File tree

15 files changed

+101
-232
lines changed

15 files changed

+101
-232
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ ext.archInfo = [
2929
],
3030
requires: [
3131
JavaVersion: isArm64 ? JavaVersion.VERSION_17 : JavaVersion.VERSION_1_8,
32-
RocksdbVersion: isArm64 ? '7.7.3' : '5.15.10'
32+
RocksdbVersion: isArm64 ? '9.7.4' : '5.15.10'
3333
],
3434
VMOptions: isArm64 ? "${rootDir}/gradle/jdk17/java-tron.vmoptions" : "${rootDir}/gradle/java-tron.vmoptions"
3535
]

‎chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,10 @@
4141
import lombok.extern.slf4j.Slf4j;
4242
import org.iq80.leveldb.DB;
4343
import org.iq80.leveldb.DBIterator;
44-
import org.iq80.leveldb.Logger;
4544
import org.iq80.leveldb.Options;
4645
import org.iq80.leveldb.ReadOptions;
4746
import org.iq80.leveldb.WriteBatch;
4847
import org.iq80.leveldb.WriteOptions;
49-
import org.slf4j.LoggerFactory;
5048
import org.tron.common.parameter.CommonParameter;
5149
import org.tron.common.storage.WriteOptionsWrapper;
5250
import org.tron.common.storage.metric.DbStat;
@@ -70,28 +68,11 @@ public class LevelDbDataSourceImpl extends DbStat implements DbSourceInter<byte[
7068
private Options options;
7169
private WriteOptions writeOptions;
7270
private ReadWriteLock resetDbLock = new ReentrantReadWriteLock();
73-
private static final org.slf4j.Logger innerLogger = LoggerFactory.getLogger(LEVELDB);
74-
private Logger leveldbLogger = new Logger() {
75-
@Override
76-
public void log(String message) {
77-
innerLogger.info("{} {}", dataBaseName, message);
78-
}
79-
};
71+
8072

8173
/**
8274
* constructor.
8375
*/
84-
public LevelDbDataSourceImpl(String parentPath, String dataBaseName, Options options,
85-
WriteOptions writeOptions) {
86-
this.parentPath = Paths.get(
87-
parentPath,
88-
CommonParameter.getInstance().getStorage().getDbDirectory()
89-
).toString();
90-
this.dataBaseName = dataBaseName;
91-
this.options = options.logger(leveldbLogger);
92-
this.writeOptions = writeOptions;
93-
initDB();
94-
}
9576

9677
public LevelDbDataSourceImpl(String parentPath, String dataBaseName) {
9778
this.parentPath = Paths.get(
@@ -100,12 +81,13 @@ public LevelDbDataSourceImpl(String parentPath, String dataBaseName) {
10081
).toString();
10182

10283
this.dataBaseName = dataBaseName;
103-
options = new Options().logger(leveldbLogger);
104-
writeOptions = new WriteOptions();
84+
this.options = StorageUtils.getOptionsByDbName(dataBaseName);
85+
this.writeOptions = new WriteOptions().sync(CommonParameter.getInstance()
86+
.getStorage().isDbSync());
87+
initDB();
10588
}
10689

107-
@Override
108-
public void initDB() {
90+
private void initDB() {
10991
resetDbLock.writeLock().lock();
11092
try {
11193
logger.debug("Init DB: {}.", dataBaseName);
@@ -443,6 +425,24 @@ public void closeDB() {
443425
}
444426
}
445427

428+
/**
429+
* Returns an iterator over the database.
430+
*
431+
* <p><b>CRITICAL:</b> The returned iterator holds native resources and <b>MUST</b> be closed
432+
* after use to prevent memory leaks. It is strongly recommended to use a try-with-resources
433+
* statement.
434+
*
435+
* <p>Example of correct usage:
436+
* <pre>{@code
437+
* try (DBIterator iterator = db.iterator()) {
438+
* while (iterator.hasNext()) {
439+
* // ... process entry
440+
* }
441+
* }
442+
* }</pre>
443+
*
444+
* @return a new database iterator that must be closed.
445+
*/
446446
@Override
447447
public org.tron.core.db.common.iterator.DBIterator iterator() {
448448
return new StoreIterator(getDBIterator());
@@ -455,7 +455,7 @@ public Stream<Entry<byte[], byte[]>> stream() {
455455
@Override
456456
public LevelDbDataSourceImpl newInstance() {
457457
return new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dataBaseName),
458-
dataBaseName, options, writeOptions);
458+
dataBaseName);
459459
}
460460

461461
private DBIterator getDBIterator() {

‎chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java‎

Lines changed: 27 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@
2222
import lombok.NoArgsConstructor;
2323
import lombok.extern.slf4j.Slf4j;
2424
import org.rocksdb.Checkpoint;
25-
import org.rocksdb.InfoLogLevel;
26-
import org.rocksdb.Logger;
2725
import org.rocksdb.Options;
2826
import org.rocksdb.ReadOptions;
2927
import org.rocksdb.RocksDB;
@@ -32,7 +30,6 @@
3230
import org.rocksdb.Status;
3331
import org.rocksdb.WriteBatch;
3432
import org.rocksdb.WriteOptions;
35-
import org.slf4j.LoggerFactory;
3633
import org.tron.common.error.TronDBException;
3734
import org.tron.common.setting.RocksDbSettings;
3835
import org.tron.common.storage.WriteOptionsWrapper;
@@ -52,26 +49,16 @@ public class RocksDbDataSourceImpl extends DbStat implements DbSourceInter<byte[
5249

5350
private String dataBaseName;
5451
private RocksDB database;
55-
private Options options;
5652
private volatile boolean alive;
5753
private String parentPath;
5854
private ReadWriteLock resetDbLock = new ReentrantReadWriteLock();
59-
private static final org.slf4j.Logger rocksDbLogger = LoggerFactory.getLogger(ROCKSDB);
6055

61-
public RocksDbDataSourceImpl(String parentPath, String name, Options options) {
56+
public RocksDbDataSourceImpl(String parentPath, String name) {
6257
this.dataBaseName = name;
6358
this.parentPath = parentPath;
64-
this.options = options;
6559
initDB();
6660
}
6761

68-
@VisibleForTesting
69-
public RocksDbDataSourceImpl(String parentPath, String name) {
70-
this.parentPath = parentPath;
71-
this.dataBaseName = name;
72-
this.options = RocksDbSettings.getOptionsByDbName(name);
73-
}
74-
7562
public Path getDbPath() {
7663
return Paths.get(parentPath, dataBaseName);
7764
}
@@ -183,7 +170,7 @@ public void setDBName(String name) {
183170
this.dataBaseName = name;
184171
}
185172

186-
public void initDB() {
173+
private void initDB() {
187174
resetDbLock.writeLock().lock();
188175
try {
189176
if (isAlive()) {
@@ -192,14 +179,8 @@ public void initDB() {
192179
if (dataBaseName == null) {
193180
throw new IllegalArgumentException("No name set to the dbStore");
194181
}
195-
options.setLogger(new Logger(options) {
196-
@Override
197-
protected void log(InfoLogLevel infoLogLevel, String logMsg) {
198-
rocksDbLogger.info("{} {}", dataBaseName, logMsg);
199-
}
200-
});
201182

202-
try {
183+
try (Options options = RocksDbSettings.getOptionsByDbName(dataBaseName)) {
203184
logger.debug("Opening database {}.", dataBaseName);
204185
final Path dbPath = getDbPath();
205186

@@ -281,6 +262,24 @@ public boolean flush() {
281262
return false;
282263
}
283264

265+
/**
266+
* Returns an iterator over the database.
267+
*
268+
* <p><b>CRITICAL:</b> The returned iterator holds native resources and <b>MUST</b> be closed
269+
* after use to prevent memory leaks. It is strongly recommended to use a try-with-resources
270+
* statement.
271+
*
272+
* <p>Example of correct usage:
273+
* <pre>{@code
274+
* try (DBIterator iterator = db.iterator()) {
275+
* while (iterator.hasNext()) {
276+
* // ... process entry
277+
* }
278+
* }
279+
* }</pre>
280+
*
281+
* @return a new database iterator that must be closed.
282+
*/
284283
@Override
285284
public org.tron.core.db.common.iterator.DBIterator iterator() {
286285
return new RockStoreIterator(getRocksIterator());
@@ -415,14 +414,15 @@ public Set<byte[]> getValuesNext(byte[] key, long limit) {
415414

416415
public void backup(String dir) throws RocksDBException {
417416
throwIfNotAlive();
418-
Checkpoint cp = Checkpoint.create(database);
419-
cp.createCheckpoint(dir + this.getDBName());
417+
try (Checkpoint cp = Checkpoint.create(database)) {
418+
cp.createCheckpoint(dir + this.getDBName());
419+
}
420420
}
421421

422422
private RocksIterator getRocksIterator() {
423-
try ( ReadOptions readOptions = new ReadOptions().setFillCache(false)) {
423+
try (ReadOptions readOptions = new ReadOptions().setFillCache(false)) {
424424
throwIfNotAlive();
425-
return database.newIterator(readOptions);
425+
return database.newIterator(readOptions);
426426
}
427427
}
428428

@@ -432,8 +432,7 @@ public boolean deleteDbBakPath(String dir) {
432432

433433
@Override
434434
public RocksDbDataSourceImpl newInstance() {
435-
return new RocksDbDataSourceImpl(parentPath, dataBaseName,
436-
this.options);
435+
return new RocksDbDataSourceImpl(parentPath, dataBaseName);
437436
}
438437

439438

‎chainbase/src/main/java/org/tron/common/utils/StorageUtils.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
package org.tron.common.utils;
22

33
import static org.tron.common.parameter.CommonParameter.ENERGY_LIMIT_HARD_FORK;
4+
import static org.tron.core.db.common.DbSourceInter.LEVELDB;
45

56
import java.io.File;
67
import org.apache.commons.lang3.StringUtils;
78
import org.iq80.leveldb.Options;
9+
import org.slf4j.LoggerFactory;
810
import org.tron.common.parameter.CommonParameter;
911
import org.tron.core.Constant;
1012

1113

1214
public class StorageUtils {
1315

16+
private static final org.slf4j.Logger levelDbLogger = LoggerFactory.getLogger(LEVELDB);
17+
1418
public static boolean getEnergyLimitHardFork() {
1519
return ENERGY_LIMIT_HARD_FORK;
1620
}
@@ -62,6 +66,7 @@ public static Options getOptionsByDbName(String dbName) {
6266
if (Constant.MARKET_PAIR_PRICE_TO_ORDER.equals(dbName)) {
6367
options.comparator(new MarketOrderPriceComparatorForLevelDB());
6468
}
69+
options.logger(message -> levelDbLogger.info("{} {}", dbName, message));
6570
return options;
6671
}
6772
}

‎chainbase/src/main/java/org/tron/core/db/TronDatabase.java‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,8 @@
88
import javax.annotation.PostConstruct;
99
import lombok.Getter;
1010
import lombok.extern.slf4j.Slf4j;
11-
import org.iq80.leveldb.WriteOptions;
1211
import org.springframework.beans.factory.annotation.Autowired;
1312
import org.tron.common.parameter.CommonParameter;
14-
import org.tron.common.storage.OptionsPicker;
1513
import org.tron.common.storage.WriteOptionsWrapper;
1614
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
1715
import org.tron.common.storage.metric.DbStatService;
@@ -24,7 +22,7 @@
2422
import org.tron.core.exception.ItemNotFoundException;
2523

2624
@Slf4j(topic = "DB")
27-
public abstract class TronDatabase<T> extends OptionsPicker implements ITronChainBase<T> {
25+
public abstract class TronDatabase<T> implements ITronChainBase<T> {
2826

2927
protected DbSourceInter<byte[]> dbSource;
3028
@Getter
@@ -40,21 +38,13 @@ protected TronDatabase(String dbName) {
4038

4139
if ("LEVELDB".equals(CommonParameter.getInstance().getStorage()
4240
.getDbEngine().toUpperCase())) {
43-
dbSource =
44-
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName),
45-
dbName,
46-
getOptionsByDbNameForLevelDB(dbName),
47-
new WriteOptions().sync(CommonParameter.getInstance()
48-
.getStorage().isDbSync()));
41+
dbSource = new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName), dbName);
4942
} else if ("ROCKSDB".equals(CommonParameter.getInstance()
5043
.getStorage().getDbEngine().toUpperCase())) {
5144
String parentName = Paths.get(StorageUtils.getOutputDirectoryByDbName(dbName),
5245
CommonParameter.getInstance().getStorage().getDbDirectory()).toString();
53-
dbSource =
54-
new RocksDbDataSourceImpl(parentName, dbName, getOptionsByDbNameForRocksDB(dbName));
46+
dbSource = new RocksDbDataSourceImpl(parentName, dbName);
5547
}
56-
57-
dbSource.initDB();
5848
}
5949

6050
@PostConstruct

‎chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,8 @@
1515
import javax.annotation.PostConstruct;
1616
import lombok.Getter;
1717
import lombok.extern.slf4j.Slf4j;
18-
import org.iq80.leveldb.WriteOptions;
1918
import org.springframework.beans.factory.annotation.Autowired;
2019
import org.tron.common.parameter.CommonParameter;
21-
import org.tron.common.storage.OptionsPicker;
2220
import org.tron.common.storage.leveldb.LevelDbDataSourceImpl;
2321
import org.tron.common.storage.metric.DbStatService;
2422
import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl;
@@ -38,7 +36,7 @@
3836

3937

4038
@Slf4j(topic = "DB")
41-
public abstract class TronStoreWithRevoking<T extends ProtoCapsule> extends OptionsPicker implements ITronChainBase<T> {
39+
public abstract class TronStoreWithRevoking<T extends ProtoCapsule> implements ITronChainBase<T> {
4240

4341
@Getter // only for unit test
4442
protected IRevokingDB revokingDB;
@@ -58,18 +56,12 @@ protected TronStoreWithRevoking(String dbName) {
5856
String dbEngine = CommonParameter.getInstance().getStorage().getDbEngine();
5957
if ("LEVELDB".equals(dbEngine.toUpperCase())) {
6058
this.db = new LevelDB(
61-
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName),
62-
dbName,
63-
getOptionsByDbNameForLevelDB(dbName),
64-
new WriteOptions().sync(CommonParameter.getInstance()
65-
.getStorage().isDbSync())));
59+
new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName), dbName));
6660
} else if ("ROCKSDB".equals(dbEngine.toUpperCase())) {
6761
String parentPath = Paths
6862
.get(StorageUtils.getOutputDirectoryByDbName(dbName), CommonParameter
6963
.getInstance().getStorage().getDbDirectory()).toString();
70-
this.db = new RocksDB(
71-
new RocksDbDataSourceImpl(parentPath,
72-
dbName, getOptionsByDbNameForRocksDB(dbName)));
64+
this.db = new RocksDB(new RocksDbDataSourceImpl(parentPath, dbName));
7365
} else {
7466
throw new RuntimeException(String.format("db engine %s is error", dbEngine));
7567
}

‎chainbase/src/main/java/org/tron/core/db/common/DbSourceInter.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,6 @@ public interface DbSourceInter<V> extends BatchSourceInter<byte[], V>,
4141

4242
void setDBName(String name);
4343

44-
void initDB();
45-
4644
boolean isAlive();
4745

4846
void closeDB();

0 commit comments

Comments
 (0)