Skip to content

Commit f02d7e7

Browse files
WillLillisWillLillis
committed
feat(test): display test results in JSON format
1 parent 6a8676f commit f02d7e7

File tree

7 files changed

+780
-400
lines changed

7 files changed

+780
-400
lines changed

‎crates/cli/src/fuzz.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,8 @@ pub fn fuzz_language_corpus(
183183

184184
if actual_output != test.output {
185185
println!("Incorrect initial parse for {test_name}");
186-
println!("{DiffKey}");
187-
println!("{}", TestDiff::new(&actual_output, &test.output, true));
186+
DiffKey::print();
187+
println!("{}", TestDiff::new(&actual_output, &test.output));
188188
println!();
189189
return false;
190190
}
@@ -276,8 +276,8 @@ pub fn fuzz_language_corpus(
276276

277277
if actual_output != test.output && !test.error {
278278
println!("Incorrect parse for {test_name} - seed {seed}");
279-
println!("{DiffKey}");
280-
println!("{}", TestDiff::new(&actual_output, &test.output, true));
279+
DiffKey::print();
280+
println!("{}", TestDiff::new(&actual_output, &test.output));
281281
println!();
282282
return false;
283283
}

‎crates/cli/src/main.rs‎

Lines changed: 126 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ use tree_sitter_cli::{
2424
input::{get_input, get_tmp_source_file, CliInput},
2525
logger,
2626
parse::{self, ParseDebugType, ParseFileOptions, ParseOutput, ParseTheme},
27-
playground, query,
27+
playground,
28+
query::{self, QueryFileOptions},
2829
tags::{self, TagsOptions},
29-
test::{self, TestOptions, TestStats},
30-
test_highlight, test_tags, util, version,
31-
version::BumpLevel,
30+
test::{self, TestOptions, TestStats, TestSummary},
31+
test_highlight, test_tags, util,
32+
version::{self, BumpLevel},
3233
wasm,
3334
};
3435
use tree_sitter_config::Config;
@@ -328,6 +329,9 @@ struct Test {
328329
/// Show only the pass-fail overview tree
329330
#[arg(long)]
330331
pub overview_only: bool,
332+
/// Output the test summary in a JSON output
333+
#[arg(long)]
334+
pub json_summary: bool,
331335
}
332336

333337
#[derive(Args)]
@@ -1150,6 +1154,28 @@ impl Parse {
11501154
}
11511155
}
11521156

1157+
/// In case an error is encountered, prints out the contents of `test_summary` and
1158+
/// propagates the error
1159+
fn check_test(
1160+
test_result: Result<()>,
1161+
test_summary: &TestSummary,
1162+
json_summary: bool,
1163+
) -> Result<()> {
1164+
if let Err(e) = test_result {
1165+
if json_summary {
1166+
let json_summary = serde_json::to_string_pretty(test_summary)
1167+
.expect("Failed to encode summary to JSON");
1168+
println!("{json_summary}");
1169+
} else {
1170+
println!("{test_summary}");
1171+
}
1172+
1173+
Err(e)?;
1174+
}
1175+
1176+
Ok(())
1177+
}
1178+
11531179
impl Test {
11541180
fn run(self, mut loader: loader::Loader, current_dir: &Path) -> Result<()> {
11551181
let config = Config::load(self.config_path)?;
@@ -1194,15 +1220,18 @@ impl Test {
11941220
parser.set_language(language)?;
11951221

11961222
let test_dir = current_dir.join("test");
1197-
let mut stats = parse::Stats::default();
1223+
let mut test_summary = TestSummary::new(
1224+
color,
1225+
stat,
1226+
self.update,
1227+
self.overview_only,
1228+
self.json_summary,
1229+
);
11981230

11991231
// Run the corpus tests. Look for them in `test/corpus`.
12001232
let test_corpus_dir = test_dir.join("corpus");
12011233
if test_corpus_dir.is_dir() {
1202-
let mut output = String::new();
1203-
let mut rates = Vec::new();
1204-
let mut opts = TestOptions {
1205-
output: &mut output,
1234+
let opts = TestOptions {
12061235
path: test_corpus_dir,
12071236
debug: self.debug,
12081237
debug_graph: self.debug_graph,
@@ -1213,51 +1242,67 @@ impl Test {
12131242
open_log: self.open_log,
12141243
languages: languages.iter().map(|(l, n)| (n.as_str(), l)).collect(),
12151244
color,
1216-
test_num: 1,
1217-
parse_rates: &mut rates,
1218-
stat_display: stat,
1219-
stats: &mut stats,
12201245
show_fields: self.show_fields,
12211246
overview_only: self.overview_only,
12221247
};
12231248

1224-
test::run_tests_at_path(&mut parser, &mut opts)?;
1225-
println!("\n{stats}");
1249+
check_test(
1250+
test::run_tests_at_path(&mut parser, &opts, &mut test_summary),
1251+
&test_summary,
1252+
self.json_summary,
1253+
)?;
1254+
test_summary.test_num = 1;
12261255
}
12271256

12281257
// Check that all of the queries are valid.
1229-
test::check_queries_at_path(language, &current_dir.join("queries"))?;
1258+
let query_dir = current_dir.join("queries");
1259+
check_test(
1260+
test::check_queries_at_path(language, &query_dir),
1261+
&test_summary,
1262+
self.json_summary,
1263+
)?;
1264+
test_summary.test_num = 1;
12301265

12311266
// Run the syntax highlighting tests.
12321267
let test_highlight_dir = test_dir.join("highlight");
12331268
if test_highlight_dir.is_dir() {
12341269
let mut highlighter = Highlighter::new();
12351270
highlighter.parser = parser;
1236-
test_highlight::test_highlights(
1237-
&loader,
1238-
&config.get()?,
1239-
&mut highlighter,
1240-
&test_highlight_dir,
1241-
color,
1271+
check_test(
1272+
test_highlight::test_highlights(
1273+
&loader,
1274+
&config.get()?,
1275+
&mut highlighter,
1276+
&test_highlight_dir,
1277+
&mut test_summary,
1278+
),
1279+
&test_summary,
1280+
self.json_summary,
12421281
)?;
12431282
parser = highlighter.parser;
1283+
test_summary.test_num = 1;
12441284
}
12451285

12461286
let test_tag_dir = test_dir.join("tags");
12471287
if test_tag_dir.is_dir() {
12481288
let mut tags_context = TagsContext::new();
12491289
tags_context.parser = parser;
1250-
test_tags::test_tags(
1251-
&loader,
1252-
&config.get()?,
1253-
&mut tags_context,
1254-
&test_tag_dir,
1255-
color,
1290+
check_test(
1291+
test_tags::test_tags(
1292+
&loader,
1293+
&config.get()?,
1294+
&mut tags_context,
1295+
&test_tag_dir,
1296+
&mut test_summary,
1297+
),
1298+
&test_summary,
1299+
self.json_summary,
12561300
)?;
1301+
test_summary.test_num = 1;
12571302
}
12581303

12591304
// For the rest of the queries, find their tests and run them
1260-
for entry in walkdir::WalkDir::new(current_dir.join("queries"))
1305+
for entry in walkdir::WalkDir::new(&query_dir)
12611306
.into_iter()
12621307
.filter_map(|e| e.ok())
12631308
.filter(|e| e.file_type().is_file())
@@ -1280,27 +1325,41 @@ impl Test {
12801325
})
12811326
.collect::<Vec<_>>();
12821327
if !entries.is_empty() {
1283-
println!("{stem}:");
1328+
test_summary.query_results.add_group(stem);
12841329
}
12851330

1286-
for entry in entries {
1331+
test_summary.test_num = 1;
1332+
let opts = QueryFileOptions::default();
1333+
for entry in &entries {
12871334
let path = entry.path();
1288-
query::query_file_at_path(
1289-
language,
1290-
path,
1291-
&path.display().to_string(),
1292-
path,
1293-
false,
1294-
None,
1295-
None,
1296-
true,
1297-
false,
1298-
false,
1299-
false,
1335+
check_test(
1336+
query::query_file_at_path(
1337+
language,
1338+
path,
1339+
&path.display().to_string(),
1340+
path,
1341+
&opts,
1342+
Some(&mut test_summary),
1343+
),
1344+
&test_summary,
1345+
self.json_summary,
13001346
)?;
13011347
}
1348+
if !entries.is_empty() {
1349+
test_summary.query_results.pop_traversal();
1350+
}
13021351
}
13031352
}
1353+
test_summary.test_num = 1;
1354+
1355+
if self.json_summary {
1356+
let json_summary = serde_json::to_string_pretty(&test_summary)
1357+
.expect("Failed to encode test summary to JSON");
1358+
println!("{json_summary}");
1359+
} else {
1360+
println!("{test_summary}");
1361+
}
1362+
13041363
Ok(())
13051364
}
13061365
}
@@ -1407,19 +1466,22 @@ impl Query {
14071466
lib_info.as_ref(),
14081467
)?;
14091468

1469+
let opts = QueryFileOptions {
1470+
ordered_captures: self.captures,
1471+
byte_range,
1472+
point_range,
1473+
quiet: self.quiet,
1474+
print_time: self.time,
1475+
stdin: false,
1476+
};
14101477
for path in paths {
14111478
query::query_file_at_path(
14121479
&language,
14131480
&path,
14141481
&path.display().to_string(),
14151482
query_path,
1416-
self.captures,
1417-
byte_range.clone(),
1418-
point_range.clone(),
1419-
self.test,
1420-
self.quiet,
1421-
self.time,
1422-
false,
1483+
&opts,
1484+
None,
14231485
)?;
14241486
}
14251487
}
@@ -1447,19 +1509,15 @@ impl Query {
14471509
.map(|(l, _)| l.clone())
14481510
.ok_or_else(|| anyhow!("No language found"))?
14491511
};
1450-
query::query_file_at_path(
1451-
language,
1452-
&path,
1453-
&name,
1454-
query_path,
1455-
self.captures,
1512+
let opts = QueryFileOptions {
1513+
ordered_captures: self.captures,
14561514
byte_range,
14571515
point_range,
1458-
self.test,
1459-
self.quiet,
1460-
self.time,
1461-
true,
1462-
)?;
1516+
quiet: self.quiet,
1517+
print_time: self.time,
1518+
stdin: true,
1519+
};
1520+
query::query_file_at_path(language, &path, &name, query_path, &opts, None)?;
14631521
fs::remove_file(path)?;
14641522
}
14651523
CliInput::Stdin(contents) => {
@@ -1469,19 +1527,15 @@ impl Query {
14691527
let path = get_tmp_source_file(&contents)?;
14701528
let language =
14711529
loader.select_language(&path, current_dir, None, lib_info.as_ref())?;
1472-
query::query_file_at_path(
1473-
&language,
1474-
&path,
1475-
"stdin",
1476-
query_path,
1477-
self.captures,
1530+
let opts = QueryFileOptions {
1531+
ordered_captures: self.captures,
14781532
byte_range,
14791533
point_range,
1480-
self.test,
1481-
self.quiet,
1482-
self.time,
1483-
true,
1484-
)?;
1534+
quiet: self.quiet,
1535+
print_time: self.time,
1536+
stdin: true,
1537+
};
1538+
query::query_file_at_path(&language, &path, "stdin", query_path, &opts, None)?;
14851539
fs::remove_file(path)?;
14861540
}
14871541
}

0 commit comments

Comments
 (0)