Skip to content

Commit cb2d613

Browse files
davidgamezdavidgamez
authored
Merge 138d550 into ada0b25 
2 parents ada0b25 + 138d550 commit cb2d613

File tree

42 files changed

+351
-477
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+351
-477
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,4 @@ app/pkg/bin/
104104
processor/notices/bin/
105105
processor/notices/tests/bin/
106106
web/service/bin/
107+
/web/service/execution_result.json

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java‎

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -61,22 +61,19 @@ public static GtfsTableContainer load(
6161
csvFile = new CsvFile(csvInputStream, gtfsFilename, settings);
6262
} catch (TextParsingException e) {
6363
noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
64-
return tableDescriptor.createContainerForInvalidStatus(
65-
GtfsTableContainer.TableStatus.INVALID_HEADERS);
64+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
6665
}
6766
if (csvFile.isEmpty()) {
6867
noticeContainer.addValidationNotice(new EmptyFileNotice(gtfsFilename));
69-
return tableDescriptor.createContainerForInvalidStatus(
70-
GtfsTableContainer.TableStatus.EMPTY_FILE);
68+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE);
7169
}
7270
final CsvHeader header = csvFile.getHeader();
7371
final ImmutableList<GtfsColumnDescriptor> columnDescriptors = tableDescriptor.getColumns();
7472
final NoticeContainer headerNotices =
7573
validateHeaders(validatorProvider, gtfsFilename, header, columnDescriptors);
7674
noticeContainer.addAll(headerNotices);
7775
if (headerNotices.hasValidationErrors()) {
78-
return tableDescriptor.createContainerForInvalidStatus(
79-
GtfsTableContainer.TableStatus.INVALID_HEADERS);
76+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
8077
}
8178
final int nColumns = columnDescriptors.size();
8279
final ImmutableMap<String, GtfsFieldLoader> fieldLoadersMap = tableDescriptor.getFieldLoaders();
@@ -133,15 +130,13 @@ public static GtfsTableContainer load(
133130
}
134131
} catch (TextParsingException e) {
135132
noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
136-
return tableDescriptor.createContainerForInvalidStatus(
137-
GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
133+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
138134
} finally {
139135
logFieldCacheStats(gtfsFilename, fieldCaches, columnDescriptors);
140136
}
141137
if (hasUnparsableRows) {
142138
logger.atSevere().log("Failed to parse some rows in %s", gtfsFilename);
143-
return tableDescriptor.createContainerForInvalidStatus(
144-
GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
139+
return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
145140
}
146141
GtfsTableContainer table =
147142
tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer);
@@ -197,14 +192,12 @@ private static void logFieldCacheStats(
197192
}
198193
}
199194

200-
public static GtfsTableContainer loadMissingFile(
201-
GtfsTableDescriptor tableDescriptor,
195+
public static GtfsContainer loadMissingFile(
196+
GtfsDescriptor tableDescriptor,
202197
ValidatorProvider validatorProvider,
203198
NoticeContainer noticeContainer) {
204199
String gtfsFilename = tableDescriptor.gtfsFilename();
205-
GtfsTableContainer table =
206-
tableDescriptor.createContainerForInvalidStatus(
207-
GtfsTableContainer.TableStatus.MISSING_FILE);
200+
GtfsContainer table = tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE);
208201
if (tableDescriptor.isRecommended()) {
209202
noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename));
210203
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package org.mobilitydata.gtfsvalidator.table;
2+
3+
import java.util.List;
4+
import java.util.Optional;
5+
6+
public abstract class GtfsContainer<T extends GtfsEntity, D extends GtfsDescriptor> {
7+
8+
private final D descriptor;
9+
private final TableStatus tableStatus;
10+
11+
public GtfsContainer(D descriptor, TableStatus tableStatus) {
12+
this.tableStatus = tableStatus;
13+
this.descriptor = descriptor;
14+
}
15+
16+
public TableStatus getTableStatus() {
17+
return tableStatus;
18+
}
19+
20+
public D getDescriptor() {
21+
return descriptor;
22+
}
23+
24+
public abstract Class<T> getEntityClass();
25+
26+
public int entityCount() {
27+
return getEntities().size();
28+
}
29+
30+
public abstract List<T> getEntities();
31+
32+
public abstract String gtfsFilename();
33+
34+
public abstract Optional<T> byTranslationKey(String recordId, String recordSubId);
35+
36+
public boolean isMissingFile() {
37+
return tableStatus == TableStatus.MISSING_FILE;
38+
}
39+
40+
public boolean isParsedSuccessfully() {
41+
switch (tableStatus) {
42+
case PARSABLE_HEADERS_AND_ROWS:
43+
return true;
44+
case MISSING_FILE:
45+
return !descriptor.isRequired();
46+
default:
47+
return false;
48+
}
49+
}
50+
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package org.mobilitydata.gtfsvalidator.table;
2+
3+
// TODO: review class name maybe GtfsFileDescriptor
4+
public abstract class GtfsDescriptor<T extends GtfsEntity> {
5+
6+
public abstract <C extends GtfsContainer> C createContainerForInvalidStatus(
7+
TableStatus tableStatus);
8+
9+
// True if the specified file is required in a feed.
10+
private boolean required;
11+
12+
private TableStatus tableStatus;
13+
14+
public abstract boolean isRecommended();
15+
16+
public abstract Class<T> getEntityClass();
17+
18+
public abstract String gtfsFilename();
19+
20+
public boolean isRequired() {
21+
return this.required;
22+
}
23+
24+
public void setRequired(boolean required) {
25+
this.required = required;
26+
}
27+
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,19 @@
1818

1919
import com.google.common.base.Ascii;
2020
import java.util.*;
21-
import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus;
2221

2322
/**
2423
* Container for a whole parsed GTFS feed with all its tables.
2524
*
26-
* <p>The tables are kept as {@code GtfsTableContainer} instances.
25+
* <p>The tables are kept as {@code GtfsContainer} instances.
2726
*/
2827
public class GtfsFeedContainer {
29-
private final Map<String, GtfsTableContainer<?>> tables = new HashMap<>();
30-
private final Map<Class<? extends GtfsTableContainer>, GtfsTableContainer<?>> tablesByClass =
28+
private final Map<String, GtfsContainer<?, ?>> tables = new HashMap<>();
29+
private final Map<Class<? extends GtfsContainer>, GtfsContainer<?, ?>> tablesByClass =
3130
new HashMap<>();
3231

33-
public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
34-
for (GtfsTableContainer<?> table : tableContainerList) {
32+
public GtfsFeedContainer(List<GtfsContainer<?, ?>> tableContainerList) {
33+
for (GtfsContainer<?, ?> table : tableContainerList) {
3534
tables.put(table.gtfsFilename(), table);
3635
tablesByClass.put(table.getClass(), table);
3736
}
@@ -49,11 +48,12 @@ public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
4948
* @param filename file name, including ".txt" extension
5049
* @return GTFS table or empty if the table is not supported by schema
5150
*/
52-
public Optional<GtfsTableContainer<?>> getTableForFilename(String filename) {
53-
return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
51+
public <T extends GtfsContainer<?, ?>> Optional<T> getTableForFilename(String filename) {
52+
return (Optional<T>)
53+
Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
5454
}
5555

56-
public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
56+
public <T extends GtfsContainer<?, ?>> T getTable(Class<T> clazz) {
5757
return (T) tablesByClass.get(clazz);
5858
}
5959

@@ -65,21 +65,21 @@ public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
6565
* @return true if all files were successfully parsed, false otherwise
6666
*/
6767
public boolean isParsedSuccessfully() {
68-
for (GtfsTableContainer<?> table : tables.values()) {
68+
for (GtfsContainer<?, ?> table : tables.values()) {
6969
if (!table.isParsedSuccessfully()) {
7070
return false;
7171
}
7272
}
7373
return true;
7474
}
7575

76-
public Collection<GtfsTableContainer<?>> getTables() {
76+
public Collection<GtfsContainer<?, ?>> getTables() {
7777
return tables.values();
7878
}
7979

8080
public String tableTotalsText() {
8181
List<String> totalList = new ArrayList<>();
82-
for (GtfsTableContainer<?> table : tables.values()) {
82+
for (GtfsContainer<?, ?> table : tables.values()) {
8383
if (table.getTableStatus() == TableStatus.MISSING_FILE
8484
&& !table.getDescriptor().isRequired()) {
8585
continue;

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java‎

Lines changed: 38 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.google.common.collect.ImmutableList;
2020
import com.google.common.flogger.FluentLogger;
2121
import java.io.InputStream;
22+
import java.lang.reflect.Modifier;
2223
import java.util.ArrayList;
2324
import java.util.Collection;
2425
import java.util.Collections;
@@ -46,7 +47,7 @@
4647
*/
4748
public class GtfsFeedLoader {
4849
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
49-
private final HashMap<String, GtfsTableDescriptor<?>> tableDescriptors = new HashMap<>();
50+
private final HashMap<String, GtfsDescriptor<?>> tableDescriptors = new HashMap<>();
5051
private int numThreads = 1;
5152

5253
/**
@@ -56,12 +57,15 @@ public class GtfsFeedLoader {
5657
private final List<Class<? extends FileValidator>> multiFileValidatorsWithParsingErrors =
5758
new ArrayList<>();
5859

59-
public GtfsFeedLoader(
60-
ImmutableList<Class<? extends GtfsTableDescriptor<?>>> tableDescriptorClasses) {
61-
for (Class<? extends GtfsTableDescriptor<?>> clazz : tableDescriptorClasses) {
62-
GtfsTableDescriptor<?> descriptor;
60+
public GtfsFeedLoader(ImmutableList<Class<? extends GtfsDescriptor<?>>> tableDescriptorClasses) {
61+
for (Class<? extends GtfsDescriptor<?>> clazz : tableDescriptorClasses) {
62+
GtfsDescriptor<?> descriptor;
6363
try {
64-
descriptor = clazz.asSubclass(GtfsTableDescriptor.class).getConstructor().newInstance();
64+
// Skipping abstract classes. Example: GtfsTableDescriptor.
65+
if (Modifier.isAbstract(clazz.getModifiers())) {
66+
continue;
67+
}
68+
descriptor = clazz.asSubclass(GtfsDescriptor.class).getConstructor().newInstance();
6569
} catch (ReflectiveOperationException e) {
6670
logger.atSevere().withCause(e).log(
6771
"Possible bug in GTFS annotation processor: expected a constructor without parameters"
@@ -73,7 +77,7 @@ public GtfsFeedLoader(
7377
}
7478
}
7579

76-
public Collection<GtfsTableDescriptor<?>> getTableDescriptors() {
80+
public Collection<GtfsDescriptor<?>> getTableDescriptors() {
7781
return Collections.unmodifiableCollection(tableDescriptors.values());
7882
}
7983

@@ -100,19 +104,36 @@ public GtfsFeedContainer loadAndValidate(
100104
Map<String, GtfsTableDescriptor<?>> remainingDescriptors =
101105
(Map<String, GtfsTableDescriptor<?>>) tableDescriptors.clone();
102106
for (String filename : gtfsInput.getFilenames()) {
103-
GtfsTableDescriptor<?> tableDescriptor = remainingDescriptors.remove(filename.toLowerCase());
107+
GtfsDescriptor<?> tableDescriptor = remainingDescriptors.remove(filename.toLowerCase());
104108
if (tableDescriptor == null) {
105109
noticeContainer.addValidationNotice(new UnknownFileNotice(filename));
106110
} else {
107111
loaderCallables.add(
108112
() -> {
109113
NoticeContainer loaderNotices = new NoticeContainer();
110-
GtfsTableContainer<?> tableContainer;
114+
GtfsContainer<?, ?> tableContainer;
111115
try (InputStream inputStream = gtfsInput.getFile(filename)) {
112116
try {
113-
tableContainer =
114-
AnyTableLoader.load(
115-
tableDescriptor, validatorProvider, inputStream, loaderNotices);
117+
if (tableDescriptor instanceof GtfsTableDescriptor) {
118+
tableContainer =
119+
AnyTableLoader.load(
120+
(GtfsTableDescriptor) tableDescriptor,
121+
validatorProvider,
122+
inputStream,
123+
loaderNotices);
124+
} else if (tableDescriptor instanceof GtfsJsonDescriptor) {
125+
tableContainer =
126+
JsonFileLoader.load(
127+
(GtfsJsonDescriptor) tableDescriptor,
128+
validatorProvider,
129+
inputStream,
130+
loaderNotices);
131+
} else {
132+
logger.atSevere().log(
133+
"Runtime exception table descriptor not supported: %s",
134+
tableDescriptor.getClass().getName());
135+
throw new RuntimeException("Table descriptor is not a supported type");
136+
}
116137
} catch (RuntimeException e) {
117138
// This handler should prevent ExecutionException for
118139
// this thread. We catch an exception here for storing
@@ -130,11 +151,10 @@ public GtfsFeedContainer loadAndValidate(
130151
});
131152
}
132153
}
133-
ArrayList<GtfsTableContainer<?>> tableContainers = new ArrayList<>();
154+
ArrayList<GtfsContainer<?, ?>> tableContainers = new ArrayList<>();
134155
tableContainers.ensureCapacity(tableDescriptors.size());
135-
for (GtfsTableDescriptor<?> tableDescriptor : remainingDescriptors.values()) {
136-
tableContainers.add(
137-
AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer));
156+
for (GtfsDescriptor<?> tableDescriptor : remainingDescriptors.values()) {
157+
AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer);
138158
}
139159
try {
140160
for (Future<TableAndNoticeContainers> futureContainer : exec.invokeAll(loaderCallables)) {
@@ -186,11 +206,10 @@ private static void addThreadExecutionError(
186206
}
187207

188208
static class TableAndNoticeContainers {
189-
final GtfsTableContainer tableContainer;
209+
final GtfsContainer tableContainer;
190210
final NoticeContainer noticeContainer;
191211

192-
public TableAndNoticeContainers(
193-
GtfsTableContainer tableContainer, NoticeContainer noticeContainer) {
212+
public TableAndNoticeContainers(GtfsContainer tableContainer, NoticeContainer noticeContainer) {
194213
this.tableContainer = tableContainer;
195214
this.noticeContainer = noticeContainer;
196215
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package org.mobilitydata.gtfsvalidator.table;
2+
3+
public abstract class GtfsJsonContainer<T extends GtfsEntity, D extends GtfsDescriptor<T>>
4+
extends GtfsContainer<T, D> {
5+
6+
public GtfsJsonContainer(D descriptor, TableStatus tableStatus) {
7+
super(descriptor, tableStatus);
8+
}
9+
}

‎core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package org.mobilitydata.gtfsvalidator.table;
2+
3+
import java.util.List;
4+
import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
5+
6+
public abstract class GtfsJsonDescriptor<T extends GtfsEntity> extends GtfsDescriptor<T> {
7+
8+
public abstract GtfsJsonContainer createContainerForEntities(
9+
List<T> entities, NoticeContainer noticeContainer);
10+
}

0 commit comments

Comments
 (0)