Skip to content

Commit 01375b1

Browse files
uglideuglide
authored
Fast json (RedisInsight#5082)
* Add simdjson to deps * Fix Hash editor layour in "new" state * Allow adding key values from file * Improve formatters aтd json handling: - Minify JSON with simdjson - Format JSON with cpp code without validation - Memorize selected formatters for each key - Remove json-tools.js - Simplify code in MultilineEditor Fix RedisInsight#5061 Fix RedisInsight#4964 Fix RedisInsight#4859
1 parent 2af6b98 commit 01375b1

20 files changed

+323
-191
lines changed

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,6 @@
1616
[submodule "3rdparty/lz4"]
1717
path = 3rdparty/lz4
1818
url = https://github.com/lz4/lz4.git
19+
[submodule "3rdparty/simdjson"]
20+
path = 3rdparty/simdjson
21+
url = https://github.com/simdjson/simdjson.git

‎3rdparty/3rdparty.pri

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ include($$PWD/pyotherside.pri)
3333
LZ4DIR = $$PWD/lz4/
3434
INCLUDEPATH += $$LZ4DIR/lib
3535

36+
#SIMDJSON
37+
SIMDJSONDIR = $$PWD/simdjson/singleheader
38+
INCLUDEPATH += $$SIMDJSONDIR/
39+
HEADERS += $$SIMDJSONDIR/simdjson.h
40+
SOURCES += $$SIMDJSONDIR/simdjson.cpp
41+
42+
3643
win32* {
3744
ZLIBDIR = $$PWD/zlib-msvc14-x64.1.2.11.7795/build/native
3845
INCLUDEPATH += $$ZLIBDIR/include

‎3rdparty/simdjson

Submodule simdjson added at ea3d4e7

‎src/app/models/key-models/keyfactory.cpp

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <qredisclient/utils/text.h>
55

66
#include <QObject>
7+
#include <QFile>
78

89
#include "hashkey.h"
910
#include "listkey.h"
@@ -119,7 +120,20 @@ void KeyFactory::submitNewKeyRequest(NewKeyRequest r) {
119120
return onRowAdded(testResp);
120121
}
121122

122-
result->addRow(r.value(), onRowAdded);
123+
auto val = r.value();
124+
125+
if (!r.valueFilePath().isEmpty() && QFile::exists(r.valueFilePath())) {
126+
QFile valueFile(r.valueFilePath());
127+
128+
if (!valueFile.open(QIODevice::ReadOnly)) {
129+
return onRowAdded(QCoreApplication::translate(
130+
"RDM", "Cannot open file with key value"));
131+
}
132+
133+
val["value"] = valueFile.readAll();
134+
}
135+
136+
result->addRow(val, onRowAdded);
123137
},
124138
onRowAdded);
125139
}

‎src/app/models/key-models/newkeyrequest.h

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,16 @@ class NewKeyRequest {
1111
Q_PROPERTY(QString keyName READ keyName WRITE setKeyName)
1212
Q_PROPERTY(QString keyType READ keyType WRITE setKeyType)
1313
Q_PROPERTY(QVariantMap value READ value WRITE setValue)
14+
Q_PROPERTY(QString valueFilePath READ valueFilePath WRITE setValueFilePath)
1415

1516
public:
1617
NewKeyRequest(QSharedPointer<RedisClient::Connection> connection, int dbIndex,
1718
std::function<void()> callback, QString keyPrefix = QString())
1819
: m_connection(connection),
1920
m_dbIndex(dbIndex),
2021
m_callback(callback),
21-
m_keyName(keyPrefix) {}
22+
m_keyName(keyPrefix),
23+
m_valueFilePath(QString()){}
2224

2325
NewKeyRequest() {}
2426

@@ -42,6 +44,10 @@ class NewKeyRequest {
4244

4345
void setValue(const QVariantMap& v) { m_value = v; }
4446

47+
QString valueFilePath() const { return m_valueFilePath; }
48+
49+
void setValueFilePath(const QString& path) { m_valueFilePath = path; }
50+
4551
QSharedPointer<RedisClient::Connection> connection() { return m_connection; }
4652

4753
void callback() const {
@@ -55,6 +61,7 @@ class NewKeyRequest {
5561
QString m_keyName;
5662
QString m_keyType;
5763
QVariantMap m_value;
64+
QString m_valueFilePath;
5865
};
5966

6067
Q_DECLARE_METATYPE(NewKeyRequest)

‎src/app/qmlutils.cpp

Lines changed: 130 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
#include <QtConcurrent>
1414
#include <QUrl>
1515

16+
#include <simdjson.h>
17+
1618
#include "apputils.h"
1719
#include "qcompress.h"
1820
#include "value-editor/largetextmodel.h"
@@ -23,7 +25,8 @@ bool QmlUtils::isBinaryString(const QVariant &value) {
2325
if (!value.canConvert(QVariant::ByteArray)) {
2426
return false;
2527
}
26-
QByteArray val = value.toByteArray();
28+
QByteArray val = value.toByteArray();
29+
2730
return isBinary(val);
2831
}
2932

@@ -44,6 +47,132 @@ QVariant QmlUtils::b64toByteArray(const QVariant &value)
4447
return QVariant(QByteArray::fromBase64(value.toString().toUtf8()));
4548
}
4649

50+
QByteArray QmlUtils::minifyJSON(const QVariant &value)
51+
{
52+
if (!value.canConvert(QVariant::ByteArray)) {
53+
return QByteArray();
54+
}
55+
56+
QByteArray val = value.toByteArray();
57+
58+
QByteArray minified;
59+
minified.resize(val.size());
60+
61+
size_t new_length{};
62+
auto error = simdjson::minify(val.data(), val.size(), minified.data(), new_length);
63+
64+
if (error != 0) {
65+
qDebug() << "Failed to minify JSON with simdjson:" << error;
66+
return QByteArray();
67+
}
68+
69+
minified.resize(new_length);
70+
71+
return minified;
72+
}
73+
74+
QByteArray QmlUtils::prettyPrintJSON(const QVariant &value)
75+
{
76+
if (!value.canConvert(QVariant::ByteArray)) {
77+
return QByteArray();
78+
}
79+
80+
QByteArray val = value.toByteArray();
81+
QByteArray result;
82+
result.reserve(val.size() * 32);
83+
84+
const QByteArray whitespace(" ");
85+
long level = 0;
86+
bool ignore_next = false;
87+
bool in_string = false;
88+
89+
// Based on https://github.com/alula/json-beautifier/blob/master/src/beautify.cpp
90+
for (auto c : qAsConst(val)) {
91+
switch (c) {
92+
case '[':
93+
case '{':
94+
if (in_string) {
95+
result.append(c);
96+
break;
97+
}
98+
level++;
99+
result.append(c);
100+
result.append("\n");
101+
for (long i = 0; i < level; i++) result.append(whitespace);
102+
break;
103+
case ']':
104+
case '}':
105+
if (in_string) {
106+
result.append(c);
107+
break;
108+
}
109+
if (level != 0) level--;
110+
result.append("\n");
111+
for (long i = 0; i < level; i++) result.append(whitespace);
112+
result.append(c);
113+
break;
114+
case ',':
115+
if (in_string) {
116+
result.append(',');
117+
break;
118+
}
119+
result.append(',');
120+
result.append("\n");
121+
for (long i = 0; i < level; i++) result.append(whitespace);
122+
break;
123+
case '\\':
124+
if (ignore_next)
125+
ignore_next = false;
126+
else
127+
ignore_next = true;
128+
result.append("\\");
129+
break;
130+
case '"':
131+
if (!ignore_next) in_string = !in_string;
132+
result.append("\"");
133+
break;
134+
case ' ':
135+
if (in_string) result.append(" ");
136+
break;
137+
case ':':
138+
result.append(":");
139+
if (!in_string) result.append(" ");
140+
break;
141+
case '\r':
142+
case '\n':
143+
break;
144+
default:
145+
if (ignore_next) ignore_next = false;
146+
result.append(c);
147+
break;
148+
}
149+
}
150+
151+
return result;
152+
}
153+
154+
bool QmlUtils::isJSON(const QVariant &value)
155+
{
156+
if (!value.canConvert(QVariant::ByteArray)) {
157+
return false;
158+
}
159+
160+
QByteArray val = value.toByteArray();
161+
int originalSize = val.size();
162+
val.resize(val.size() + simdjson::SIMDJSON_PADDING);
163+
164+
simdjson::dom::parser parser;
165+
simdjson::dom::element data;
166+
auto error = parser.parse(val.data(), originalSize, false).get(data);
167+
168+
if (error != simdjson::SUCCESS && error != simdjson::NUMBER_ERROR) {
169+
qDebug() << "JSON is not valid:" << simdjson::error_message(error);
170+
return false;
171+
}
172+
173+
return true;
174+
}
175+
47176
QVariant QmlUtils::decompress(const QVariant &value) {
48177
if (!value.canConvert(QVariant::ByteArray)) {
49178
return 0;

‎src/app/qmlutils.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,13 @@
99
class QmlUtils : public QObject
1010
{
1111
Q_OBJECT
12-
public:
12+
public:
1313
Q_INVOKABLE bool isBinaryString(const QVariant &value);
1414
Q_INVOKABLE long binaryStringLength(const QVariant &value);
1515
Q_INVOKABLE QVariant b64toByteArray(const QVariant &value);
16+
Q_INVOKABLE QByteArray minifyJSON(const QVariant &value);
17+
Q_INVOKABLE QByteArray prettyPrintJSON(const QVariant &value);
18+
Q_INVOKABLE bool isJSON(const QVariant &value);
1619

1720
Q_INVOKABLE unsigned isCompressed(const QVariant &value);
1821
Q_INVOKABLE QVariant decompress(const QVariant &value);

‎src/qml/GlobalSettings.qml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ Dialog {
197197
Layout.preferredHeight: 30
198198

199199
min: 1
200-
max: 2000000
200+
max: 20000000
201201
value: 150000
202202
label: qsTranslate("RDM","Maximum Formatted Value Size")
203203
description: qsTranslate("RDM", "Size in bytes")

‎src/qml/app.qml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ ApplicationWindow {
7272
property int valueSizeLimit: 150000
7373
}
7474

75+
Settings {
76+
id: defaultFormatterSettings
77+
category: "formatter_overrides"
78+
}
79+
7580
SystemPalette {
7681
id: sysPalette
7782
}

‎src/qml/common/BetterComboBox.qml

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,15 @@ ComboBox {
1515
palette.dark: sysPalette.dark
1616
palette.window: sysPalette.window
1717

18-
function selectItem(txt) {
18+
function selectItem(txt) {
19+
var res = _select(txt);
20+
21+
if (res >= 0) {
22+
activated(res)
23+
}
24+
}
25+
26+
function _select(txt) {
1927
var index = find(txt)
2028

2129
console.log("Index:", index)
@@ -24,6 +32,6 @@ ComboBox {
2432
currentIndex = index;
2533
}
2634

27-
activated(index)
35+
return index
2836
}
2937
}

‎src/qml/qml.qrc

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@
3636
<file>GlobalSettings.qml</file>
3737
<file>settings/BoolOption.qml</file>
3838
<file>settings/IntOption.qml</file>
39-
<file>bulk-operations/BulkOperationsDialog.qml</file>
40-
<file>value-editor/editors/formatters/json-tools.js</file>
39+
<file>bulk-operations/BulkOperationsDialog.qml</file>
4140
<file>settings/ComboboxOption.qml</file>
4241
<file>server-info/ServerInfoTabs.qml</file>
4342
<file>common/platformutils.js</file>

‎src/qml/value-editor/AddKeyDialog.qml

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ BetterDialog {
4040
model: Editor.getSupportedKeyTypes()
4141
Layout.fillWidth: true
4242
objectName: "rdm_add_key_type_field"
43+
44+
onCurrentIndexChanged: {
45+
if (valueAddEditor.item.keyType !== undefined)
46+
valueAddEditor.item.keyType = typeSelector.model[typeSelector.currentIndex]
47+
}
4348
}
4449

4550
Loader {
@@ -60,6 +65,18 @@ BetterDialog {
6065
}
6166
}
6267

68+
BetterLabel { text: qsTranslate("RDM", "Or Import Value from the file") + ":" }
69+
70+
FilePathInput {
71+
id: valueFilePath
72+
objectName: "rdm_add_key_value_file"
73+
Layout.fillWidth: true
74+
placeholderText: qsTranslate("RDM","(Optional) Any file")
75+
nameFilters: [ "Any file (*)" ]
76+
title: qsTranslate("RDM","Select file with value")
77+
path: ""
78+
}
79+
6380
RowLayout {
6481
Layout.fillWidth: true
6582
Layout.minimumHeight: 40
@@ -70,18 +87,26 @@ BetterDialog {
7087
objectName: "rdm_add_key_save_btn"
7188
text: qsTranslate("RDM","Save")
7289

90+
function submitNewKeyRequest() {
91+
root.request.keyName = newKeyName.text
92+
root.request.keyType = typeSelector.model[typeSelector.currentIndex]
93+
root.request.value = valueAddEditor.item.getValue()
94+
root.request.valueFilePath = valueFilePath.path
95+
keyFactory.submitNewKeyRequest(root.request)
96+
}
97+
98+
7399
onClicked: {
74100
if (!valueAddEditor.item)
75101
return
76102

103+
valueAddEditor.item.validateVal = (valueFilePath.path === "")
104+
77105
valueAddEditor.item.validateValue(function (result) {
78106
if (!result)
79107
return;
80108

81-
root.request.keyName = newKeyName.text
82-
root.request.keyType = typeSelector.model[typeSelector.currentIndex]
83-
root.request.value = valueAddEditor.item.getValue()
84-
keyFactory.submitNewKeyRequest(root.request)
109+
submitNewKeyRequest();
85110
})
86111
}
87112

@@ -92,6 +117,7 @@ BetterDialog {
92117
root.request = null
93118
valueAddEditor.item.reset()
94119
valueAddEditor.item.initEmpty()
120+
valueFilePath.path = ""
95121
root.close()
96122
}
97123

‎src/qml/value-editor/editors/AbstractEditor.qml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ ColumnLayout {
88

99
state: "edit"
1010

11+
property bool validateVal: true
12+
1113
states: [
1214
State { name: "new"}, // Creating new key
1315
State { name: "add"}, // Adding new value to existing key

0 commit comments

Comments
 (0)