Skip to content

Commit 78eee27

Browse files
jcpitrejcpitre
authored
Merge ee77597 into 63dab81 
2 parents 63dab81 + ee77597 commit 78eee27

File tree

49 files changed

+932
-348
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+932
-348
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,5 +104,6 @@ app/pkg/bin/
104104
processor/notices/bin/
105105
processor/notices/tests/bin/
106106
web/service/bin/
107+
/web/service/execution_result.json
107108

108109
RULES.md

‎core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import java.util.Optional;
3333
import java.util.TreeMap;
3434
import java.util.logging.Level;
35+
import org.mobilitydata.gtfsvalidator.annotation.GtfsJson;
3536
import org.mobilitydata.gtfsvalidator.annotation.GtfsTable;
3637
import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice;
3738
import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef;
@@ -125,7 +126,12 @@ public static NoticeDocComments loadComments(Class<?> noticeClass) {
125126
private static ReferencesSchema generateReferences(GtfsValidationNotice noticeAnnotation) {
126127
ReferencesSchema schema = new ReferencesSchema();
127128
Arrays.stream(noticeAnnotation.files().value())
128-
.map(NoticeSchemaGenerator::getFileIdForTableClass)
129+
.map(
130+
// Both Table and Json annotations specify a file name, collect them all.
131+
fileClass -> {
132+
Optional<String> fileId = getFileIdForTableClass(fileClass);
133+
return fileId.or(() -> getFileIdForJsonClass(fileClass));
134+
})
129135
.flatMap(Optional::stream)
130136
.forEach(schema::addFileReference);
131137
Arrays.stream(noticeAnnotation.bestPractices().value())
@@ -146,6 +152,11 @@ private static Optional<String> getFileIdForTableClass(Class<? extends GtfsEntit
146152
return Optional.ofNullable(table).map(GtfsTable::value);
147153
}
148154

155+
private static Optional<String> getFileIdForJsonClass(Class<? extends GtfsEntity> entityClass) {
156+
GtfsJson annotation = entityClass.getAnnotation(GtfsJson.class);
157+
return Optional.ofNullable(annotation).map(GtfsJson::value);
158+
}
159+
149160
private static UrlReference convertUrlRef(UrlRef ref) {
150161
return new UrlReference(ref.label(), ref.url());
151162
}

core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java renamed to core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java

Lines changed: 25 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -7,48 +7,44 @@
77
import com.univocity.parsers.csv.CsvParserSettings;
88
import java.io.InputStream;
99
import java.util.ArrayList;
10-
import java.util.Collections;
1110
import java.util.List;
1211
import java.util.Optional;
1312
import java.util.stream.Collectors;
13+
import javax.annotation.Nonnull;
1414
import javax.annotation.Nullable;
1515
import org.mobilitydata.gtfsvalidator.notice.CsvParsingFailedNotice;
1616
import org.mobilitydata.gtfsvalidator.notice.EmptyFileNotice;
17-
import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice;
18-
import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice;
1917
import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
2018
import org.mobilitydata.gtfsvalidator.parsing.CsvFile;
2119
import org.mobilitydata.gtfsvalidator.parsing.CsvHeader;
2220
import org.mobilitydata.gtfsvalidator.parsing.CsvRow;
2321
import org.mobilitydata.gtfsvalidator.parsing.FieldCache;
2422
import org.mobilitydata.gtfsvalidator.parsing.RowParser;
25-
import org.mobilitydata.gtfsvalidator.validator.FileValidator;
2623
import org.mobilitydata.gtfsvalidator.validator.SingleEntityValidator;
2724
import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider;
2825
import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil;
2926

30-
public final class AnyTableLoader {
27+
/** This class loads csv files specifically. */
28+
public final class CsvFileLoader extends TableLoader {
3129

32-
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
33-
private static final List<Class<? extends FileValidator>> singleFileValidatorsWithParsingErrors =
34-
new ArrayList<>();
30+
private CsvFileLoader() {}
31+
// Create the singleton and add a method to obtain it
32+
private static final CsvFileLoader INSTANCE = new CsvFileLoader();
3533

36-
private static final List<Class<? extends SingleEntityValidator>>
37-
singleEntityValidatorsWithParsingErrors = new ArrayList<>();
38-
39-
public List<Class<? extends FileValidator>> getValidatorsWithParsingErrors() {
40-
return Collections.unmodifiableList(singleFileValidatorsWithParsingErrors);
34+
@Nonnull
35+
public static CsvFileLoader getInstance() {
36+
return INSTANCE;
4137
}
4238

43-
public List<Class<? extends SingleEntityValidator>> getSingleEntityValidatorsWithParsingErrors() {
44-
return Collections.unmodifiableList(singleEntityValidatorsWithParsingErrors);
45-
}
39+
private final FluentLogger logger = FluentLogger.forEnclosingClass();
4640

47-
public static GtfsTableContainer load(
48-
GtfsTableDescriptor tableDescriptor,
41+
@Override
42+
public GtfsEntityContainer<?, ?> load(
43+
GtfsFileDescriptor fileDescriptor,
4944
ValidatorProvider validatorProvider,
5045
InputStream csvInputStream,
5146
NoticeContainer noticeContainer) {
47+
GtfsTableDescriptor tableDescriptor = (GtfsTableDescriptor) fileDescriptor;
5248
final String gtfsFilename = tableDescriptor.gtfsFilename();
5349

5450
CsvFile csvFile;
@@ -61,22 +57,19 @@ public static GtfsTableContainer load(
6157
csvFile = new CsvFile(csvInputStream, gtfsFilename, settings);
6258
} catch (TextParsingException e) {
6359
noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
64-
return tableDescriptor.createContainerForInvalidStatus(
65-
GtfsTableContainer.TableStatus.INVALID_HEADERS);
60+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
6661
}
6762
if (csvFile.isEmpty()) {
6863
noticeContainer.addValidationNotice(new EmptyFileNotice(gtfsFilename));
69-
return tableDescriptor.createContainerForInvalidStatus(
70-
GtfsTableContainer.TableStatus.EMPTY_FILE);
64+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE);
7165
}
7266
final CsvHeader header = csvFile.getHeader();
7367
final ImmutableList<GtfsColumnDescriptor> columnDescriptors = tableDescriptor.getColumns();
7468
final NoticeContainer headerNotices =
7569
validateHeaders(validatorProvider, gtfsFilename, header, columnDescriptors);
7670
noticeContainer.addAll(headerNotices);
7771
if (headerNotices.hasValidationErrors()) {
78-
return tableDescriptor.createContainerForInvalidStatus(
79-
GtfsTableContainer.TableStatus.INVALID_HEADERS);
72+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
8073
}
8174
final int nColumns = columnDescriptors.size();
8275
final ImmutableMap<String, GtfsFieldLoader> fieldLoadersMap = tableDescriptor.getFieldLoaders();
@@ -99,8 +92,8 @@ public static GtfsTableContainer load(
9992
final List<GtfsEntity> entities = new ArrayList<>();
10093
boolean hasUnparsableRows = false;
10194
final List<SingleEntityValidator<GtfsEntity>> singleEntityValidators =
102-
validatorProvider.createSingleEntityValidators(
103-
tableDescriptor.getEntityClass(), singleEntityValidatorsWithParsingErrors::add);
95+
createSingleEntityValidators(tableDescriptor.getEntityClass(), validatorProvider);
96+
10497
try {
10598
for (CsvRow row : csvFile) {
10699
if (row.getRowNumber() % 200000 == 0) {
@@ -133,26 +126,23 @@ public static GtfsTableContainer load(
133126
}
134127
} catch (TextParsingException e) {
135128
noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
136-
return tableDescriptor.createContainerForInvalidStatus(
137-
GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
129+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
138130
} finally {
139131
logFieldCacheStats(gtfsFilename, fieldCaches, columnDescriptors);
140132
}
141133
if (hasUnparsableRows) {
142134
logger.atSevere().log("Failed to parse some rows in %s", gtfsFilename);
143-
return tableDescriptor.createContainerForInvalidStatus(
144-
GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
135+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
145136
}
146137
GtfsTableContainer table =
147138
tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer);
139+
148140
ValidatorUtil.invokeSingleFileValidators(
149-
validatorProvider.createSingleFileValidators(
150-
table, singleFileValidatorsWithParsingErrors::add),
151-
noticeContainer);
141+
createSingleFileValidators(table, validatorProvider), noticeContainer);
152142
return table;
153143
}
154144

155-
private static NoticeContainer validateHeaders(
145+
private NoticeContainer validateHeaders(
156146
ValidatorProvider validatorProvider,
157147
String gtfsFilename,
158148
CsvHeader header,
@@ -178,7 +168,7 @@ private static NoticeContainer validateHeaders(
178168
return headerNotices;
179169
}
180170

181-
private static void logFieldCacheStats(
171+
private void logFieldCacheStats(
182172
String gtfsFilename,
183173
FieldCache[] fieldCaches,
184174
ImmutableList<GtfsColumnDescriptor> columnDescriptors) {
@@ -196,25 +186,4 @@ private static void logFieldCacheStats(
196186
}
197187
}
198188
}
199-
200-
public static GtfsTableContainer loadMissingFile(
201-
GtfsTableDescriptor tableDescriptor,
202-
ValidatorProvider validatorProvider,
203-
NoticeContainer noticeContainer) {
204-
String gtfsFilename = tableDescriptor.gtfsFilename();
205-
GtfsTableContainer table =
206-
tableDescriptor.createContainerForInvalidStatus(
207-
GtfsTableContainer.TableStatus.MISSING_FILE);
208-
if (tableDescriptor.isRecommended()) {
209-
noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename));
210-
}
211-
if (tableDescriptor.isRequired()) {
212-
noticeContainer.addValidationNotice(new MissingRequiredFileNotice(gtfsFilename));
213-
}
214-
ValidatorUtil.invokeSingleFileValidators(
215-
validatorProvider.createSingleFileValidators(
216-
table, singleFileValidatorsWithParsingErrors::add),
217-
noticeContainer);
218-
return table;
219-
}
220189
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package org.mobilitydata.gtfsvalidator.table;
2+
3+
import java.util.List;
4+
import java.util.Optional;
5+
6+
/**
7+
* This class is the parent of containers holding table (csv) entities and containers holding JSON
8+
* entities
9+
*
10+
* @param <T> The entity for this container (e.g. GtfsCalendarDate or GtfsGeojsonFeature )
11+
* @param <D> The descriptor for the file for the container (e.g. GtfsCalendarDateTableDescriptor or
12+
* GtfsGeojsonFileDescriptor)
13+
*/
14+
public abstract class GtfsEntityContainer<T extends GtfsEntity, D extends GtfsFileDescriptor> {
15+
16+
private final D descriptor;
17+
private final TableStatus tableStatus;
18+
19+
public GtfsEntityContainer(D descriptor, TableStatus tableStatus) {
20+
this.tableStatus = tableStatus;
21+
this.descriptor = descriptor;
22+
}
23+
24+
public TableStatus getTableStatus() {
25+
return tableStatus;
26+
}
27+
28+
public D getDescriptor() {
29+
return descriptor;
30+
}
31+
32+
public abstract Class<T> getEntityClass();
33+
34+
public int entityCount() {
35+
return getEntities().size();
36+
}
37+
38+
public abstract List<T> getEntities();
39+
40+
public abstract String gtfsFilename();
41+
42+
public abstract Optional<T> byTranslationKey(String recordId, String recordSubId);
43+
44+
public boolean isMissingFile() {
45+
return tableStatus == TableStatus.MISSING_FILE;
46+
}
47+
48+
public boolean isParsedSuccessfully() {
49+
switch (tableStatus) {
50+
case PARSABLE_HEADERS_AND_ROWS:
51+
return true;
52+
case MISSING_FILE:
53+
return !descriptor.isRequired();
54+
default:
55+
return false;
56+
}
57+
}
58+
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,19 @@
1818

1919
import com.google.common.base.Ascii;
2020
import java.util.*;
21-
import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus;
2221

2322
/**
2423
* Container for a whole parsed GTFS feed with all its tables.
2524
*
26-
* <p>The tables are kept as {@code GtfsTableContainer} instances.
25+
* <p>The tables are kept as {@link GtfsEntityContainer} instances.
2726
*/
2827
public class GtfsFeedContainer {
29-
private final Map<String, GtfsTableContainer<?>> tables = new HashMap<>();
30-
private final Map<Class<? extends GtfsTableContainer>, GtfsTableContainer<?>> tablesByClass =
28+
private final Map<String, GtfsEntityContainer<?, ?>> tables = new HashMap<>();
29+
private final Map<Class<? extends GtfsEntityContainer>, GtfsEntityContainer<?, ?>> tablesByClass =
3130
new HashMap<>();
3231

33-
public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
34-
for (GtfsTableContainer<?> table : tableContainerList) {
32+
public GtfsFeedContainer(List<GtfsEntityContainer<?, ?>> tableContainerList) {
33+
for (GtfsEntityContainer<?, ?> table : tableContainerList) {
3534
tables.put(table.gtfsFilename(), table);
3635
tablesByClass.put(table.getClass(), table);
3736
}
@@ -49,11 +48,12 @@ public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
4948
* @param filename file name, including ".txt" extension
5049
* @return GTFS table or empty if the table is not supported by schema
5150
*/
52-
public Optional<GtfsTableContainer<?>> getTableForFilename(String filename) {
53-
return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
51+
public <T extends GtfsEntityContainer<?, ?>> Optional<T> getTableForFilename(String filename) {
52+
return (Optional<T>)
53+
Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
5454
}
5555

56-
public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
56+
public <T extends GtfsEntityContainer<?, ?>> T getTable(Class<T> clazz) {
5757
return (T) tablesByClass.get(clazz);
5858
}
5959

@@ -65,21 +65,21 @@ public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
6565
* @return true if all files were successfully parsed, false otherwise
6666
*/
6767
public boolean isParsedSuccessfully() {
68-
for (GtfsTableContainer<?> table : tables.values()) {
68+
for (GtfsEntityContainer<?, ?> table : tables.values()) {
6969
if (!table.isParsedSuccessfully()) {
7070
return false;
7171
}
7272
}
7373
return true;
7474
}
7575

76-
public Collection<GtfsTableContainer<?>> getTables() {
76+
public Collection<GtfsEntityContainer<?, ?>> getTables() {
7777
return tables.values();
7878
}
7979

8080
public String tableTotalsText() {
8181
List<String> totalList = new ArrayList<>();
82-
for (GtfsTableContainer<?> table : tables.values()) {
82+
for (GtfsEntityContainer<?, ?> table : tables.values()) {
8383
if (table.getTableStatus() == TableStatus.MISSING_FILE
8484
&& !table.getDescriptor().isRequired()) {
8585
continue;

0 commit comments

Comments
 (0)