Skip to content

Fix Deep Learning header classification inconsistency #1211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 21 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
d95a1ac
first attempt to fix tables on the fly
lfoppiano Dec 4, 2024
b012665
revise table validation, apply check to figures, move code outside th…
lfoppiano Dec 6, 2024
d2ca7c9
fix figure validation
lfoppiano Dec 6, 2024
575c2fe
move test related to LabelUtils outside
lfoppiano Dec 6, 2024
7780804
fix scope
lfoppiano Dec 6, 2024
e0e217d
remove unnecessary layout tokens list, renaming stuff
lfoppiano Dec 6, 2024
facc35e
fix index mismatch
lfoppiano Dec 6, 2024
17ad425
fix wrong label adjustment
lfoppiano Dec 6, 2024
c830231
fix wrong upper indexes, cleanup
lfoppiano Dec 7, 2024
9c77194
Merge branch 'refs/heads/master' into bugfix/fix-dropped-misclassifie…
lfoppiano Dec 16, 2024
2f0661b
cosmetics
lfoppiano Dec 16, 2024
2f04ccd
add tests for labelling adjustment
lfoppiano Dec 16, 2024
f5eb758
tests and refactoring in smaller pieces
lfoppiano Dec 17, 2024
81a1691
fix typo
lfoppiano Dec 17, 2024
3778a6e
add test for table token consolidation
lfoppiano Dec 17, 2024
21f85c9
rename methods for better clarity, move utility methods in Kotlin
lfoppiano Dec 18, 2024
1fa792c
fix header sequence with delft to revert any labels that do not have …
lfoppiano Dec 18, 2024
cbbe460
renaming
lfoppiano Dec 19, 2024
f036e0a
allow a loose approach to gather table/figure starting token when the…
lfoppiano Dec 19, 2024
09c824d
improve the way the candidate identification for bad figures/tables i…
lfoppiano Dec 19, 2024
77cf7a2
Merge branch 'bugfix/fix-dropped-misclassified-text' into bugfix/fix-…
lfoppiano Dec 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix typo
  • Loading branch information
lfoppiano committed Dec 17, 2024
commit 81a1691008a0c232bbbf878c0defb7a6c65ef346
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ static String revertResultsForBadItems(List<? extends Figure> badFiguresOrTables
for (Figure badItem : badFiguresOrTables) {
// Find the index of the first layoutToken of the table in the tokenization
List<LayoutToken> layoutTokenItem = badItem.getLayoutTokens();
List<Integer> candidateIndexes = findCandiateIndex(layoutTokenItem, labelledResultsAsList, itemLabel);
List<Integer> candidateIndexes = findCandidateIndex(layoutTokenItem, labelledResultsAsList, itemLabel);
if (candidateIndexes.isEmpty()) {
LOGGER.info("Cannot find the candidate index for fixing the tables.");
continue;
Expand Down Expand Up @@ -441,7 +441,7 @@ static int consolidateResultCandidateThroughSequence(List<Integer> candidateInde
}

@NotNull
static List<Integer> findCandiateIndex(List<LayoutToken> layoutTokenItem, List<List<String>> labelledResultsAsList, String itemLabel) {
static List<Integer> findCandidateIndex(List<LayoutToken> layoutTokenItem, List<List<String>> labelledResultsAsList, String itemLabel) {
LayoutToken firstLayoutTokenItem = layoutTokenItem.get(0);

List<Integer> candidateIndexes = IntStream.range(0, labelledResultsAsList.size())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@ package org.grobid.core.engines

import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.tuple.Triple
import org.easymock.EasyMock.createMock
import org.grobid.core.analyzers.GrobidAnalyzer
import org.grobid.core.document.Document
import org.grobid.core.document.DocumentSource
import org.grobid.core.engines.label.TaggingLabels.TABLE_LABEL
import org.grobid.core.factory.GrobidFactory
import org.grobid.core.main.LibraryLoader
Expand Down Expand Up @@ -234,10 +231,7 @@ class FullTextParserTest {
}

@Test
fun testFindCandidates() {
var mockDocumentSource = createMock<DocumentSource>(DocumentSource::class.java)
var document = Document.createFromText("")

fun testFindCandidates_shouldFindMultipleResults() {
// i need to prepare a sequence where there might be multiple matches,
// and then verify that the sequence is correctly used for discrimination
var sequence = "This article solves the problem where some of our interaction are fauly. " +
Expand Down Expand Up @@ -284,7 +278,7 @@ class FullTextParserTest {
println(wapitiResult)

val table1Tokens = tokens.subList(25, 61)
val foundCandidateIndex = FullTextParser.findCandiateIndex(table1Tokens, labelledResultsAsList, TABLE_LABEL)
val foundCandidateIndex = FullTextParser.findCandidateIndex(table1Tokens, labelledResultsAsList, TABLE_LABEL)

assertThat(foundCandidateIndex, hasSize(3))
assertThat(foundCandidateIndex.get(0), `is`(13))
Expand Down
Loading