Skip to content

Commit 32f8610

Browse files
mcpiercemcpierce
committed
Added a batch process for rescraping comic books [#2133]
Also standardized using constants for job names and steps, and removed the old job listener for importing comic books.
1 parent 4698e87 commit 32f8610

File tree

75 files changed

+1525
-142
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+1525
-142
lines changed

‎comixed-app/src/main/resources/application.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ comixed.batch.mark-blocked-pages.period=60000
9494
comixed.batch.mark-blocked-pages.chunk-size=1
9595
comixed.batch.organize-library.period=60000
9696
comixed.batch.organize-library.chunk-size=1
97+
comixed.batch.scrape-metadata.chunk-size=10
9798
comixed.batch.update-metadata.period=60000
9899
comixed.batch.update-metadata.chunk-size=10
99100

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/OrganizeLibraryConfiguration.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,12 @@
5454
@Log4j2
5555
public class OrganizeLibraryConfiguration {
5656
public static final String ORGANIZE_LIBRARY_JOB = "organizeLibraryJob";
57-
public static final String JOB_ORGANIZATION_TIME_STARTED = "job.organization.time-started";
58-
public static final String JOB_ORGANIZATION_DELETE_REMOVED_COMIC_FILES =
57+
public static final String ORGANIZE_LIBRARY_JOB_TIME_STARTED = "job.organization.time-started";
58+
public static final String ORGANIZE_LIBRARY_JOB_DELETE_REMOVED_COMIC_FILES =
5959
"job.organization.delete-removed-comic-files";
60-
public static final String JOB_ORGANIZATION_TARGET_DIRECTORY =
60+
public static final String ORGANIZE_LIBRARY_JOB_TARGET_DIRECTORY =
6161
"job.organization.target-directory";
62-
public static final String JOB_ORGANIZATION_RENAMING_RULE = "job.organization.renaming-rule";
62+
public static final String ORGANIZE_LIBRARY_JOB_RENAMING_RULE = "job.organization.renaming-rule";
6363

6464
@Value("${comixed.batch.organize-library.chunk-size:1}")
6565
private int chunkSize;

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/ProcessComicBooksConfiguration.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@
4545
@Log4j2
4646
public class ProcessComicBooksConfiguration {
4747
public static final String PROCESS_COMIC_BOOKS_JOB = "processComicBooksJob";
48-
public static final String PROCESS_COMIC_BOOKS_STARTED_JOB = "job.process-comic-books.started";
48+
public static final String PROCESS_COMIC_BOOKS_STARTED_JOB =
49+
"job.process-comic-books.time-started";
4950

5051
@Value("${comixed.batch.process-comic-books.chunk-size:10}")
5152
private int chunkSize;

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/PurgeLibraryConfiguration.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848
@Log4j2
4949
public class PurgeLibraryConfiguration {
5050
public static final String PURGE_LIBRARY_JOB = "purgeLibraryJob";
51-
public static final String JOB_PURGE_LIBRARY_START = "job.purge-library.started";
51+
public static final String PURGE_LIBRARY_JOB_TIME_STARTED = "job.purge-library.time-started";
5252

5353
@Value("${comixed.batch.purge-library.chunk-size:1}")
5454
private int chunkSize;

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/RecreateComicFilesConfiguration.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@
4646
@Log4j2
4747
public class RecreateComicFilesConfiguration {
4848
public static final String RECREATE_COMIC_FILES_JOB = "recreateComicFilesJob";
49-
public static final String JOB_RECREATE_COMICS_STARTED = "job.recreate-comic.started";
49+
public static final String RECREATE_COMIC_FILES_JOB_TIME_STARTED =
50+
"job.recreate-comic.time-started";
5051

5152
@Value("${comixed.batch.recreate-comic-files.chunk-size:1}")
5253
private int chunkSize;

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/ScrapeMetadataConfiguration.java‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* ComiXed - A digital comic book library management application.
3+
* Copyright (C) 2024, The ComiXed Project
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses>
17+
*/
18+
19+
package org.comixedproject.batch.comicbooks;
20+
21+
import lombok.extern.log4j.Log4j2;
22+
import org.comixedproject.batch.comicbooks.listeners.ScrapeMetadataChunkListener;
23+
import org.comixedproject.batch.comicbooks.listeners.ScrapeMetadataJobListener;
24+
import org.comixedproject.batch.comicbooks.processors.ScrapeMetadataProcessor;
25+
import org.comixedproject.batch.comicbooks.readers.ScrapeMetadataReader;
26+
import org.comixedproject.batch.writers.NoopWriter;
27+
import org.comixedproject.model.comicbooks.ComicBook;
28+
import org.springframework.batch.core.Job;
29+
import org.springframework.batch.core.Step;
30+
import org.springframework.batch.core.job.builder.JobBuilder;
31+
import org.springframework.batch.core.launch.support.RunIdIncrementer;
32+
import org.springframework.batch.core.repository.JobRepository;
33+
import org.springframework.batch.core.step.builder.StepBuilder;
34+
import org.springframework.beans.factory.annotation.Qualifier;
35+
import org.springframework.beans.factory.annotation.Value;
36+
import org.springframework.context.annotation.Bean;
37+
import org.springframework.context.annotation.Configuration;
38+
import org.springframework.transaction.PlatformTransactionManager;
39+
40+
/**
41+
* <code>ScrapeMetadataConfiguration</code> defines a batch process that scrapes comic book metadata
42+
* in the background.
43+
*
44+
* @author Darryl L. Pierce
45+
*/
46+
@Configuration
47+
@Log4j2
48+
public class ScrapeMetadataConfiguration {
49+
public static final String SCRAPE_METADATA_JOB = "scrapeMetadataJob";
50+
public static final String SCRAPE_METADATA_JOB_TIME_STARTED = "job.scrape-metadata.time-started";
51+
public static final String SCRAPE_METADATA_JOB_ERROR_THRESHOLD =
52+
"job.scrape-metadata.error-threshold";
53+
public static final String SCRAPE_METADATA_STEP = "scrape-metadata-step";
54+
55+
@Value("${comixed.batch.scrape-metadata.chunk-size:10}")
56+
private int chunkSize;
57+
58+
/**
59+
* Returns the scrape metadata job.
60+
*
61+
* @param jobRepository the job repository
62+
* @param jobListener the job listener
63+
* @param scrapeMetadataStep the scrape metadata step
64+
* @return the job
65+
*/
66+
@Bean(name = SCRAPE_METADATA_JOB)
67+
public Job scrapeMetadataJob(
68+
final JobRepository jobRepository,
69+
final ScrapeMetadataJobListener jobListener,
70+
@Qualifier("scrapeMetadataStep") final Step scrapeMetadataStep) {
71+
return new JobBuilder(SCRAPE_METADATA_JOB, jobRepository)
72+
.incrementer(new RunIdIncrementer())
73+
.listener(jobListener)
74+
.start(scrapeMetadataStep)
75+
.build();
76+
}
77+
78+
/**
79+
* Returns the scrape metadata step.
80+
*
81+
* @param jobRepository the step factory
82+
* @param platformTransactionManager the transaction manager
83+
* @param reader the reader
84+
* @param processor the processor
85+
* @param writer the writer
86+
* @param chunkListener the chunk listener
87+
* @return the step
88+
*/
89+
@Bean(name = "scrapeMetadataStep")
90+
public Step scrapeMetadataStep(
91+
final JobRepository jobRepository,
92+
final PlatformTransactionManager platformTransactionManager,
93+
final ScrapeMetadataReader reader,
94+
final ScrapeMetadataProcessor processor,
95+
final NoopWriter<ComicBook> writer,
96+
final ScrapeMetadataChunkListener chunkListener) {
97+
return new StepBuilder("scrapeMetadataStep", jobRepository)
98+
.<ComicBook, ComicBook>chunk(this.chunkSize, platformTransactionManager)
99+
.reader(reader)
100+
.processor(processor)
101+
.writer(writer)
102+
.listener(chunkListener)
103+
.build();
104+
}
105+
}

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/UpdateComicBooksConfiguration.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,22 +44,23 @@
4444
@Configuration
4545
@Log4j2
4646
public class UpdateComicBooksConfiguration {
47-
public static final String JOB_UPDATE_COMICBOOKS_STARTED = "job.update-comic.started";
48-
public static final String JOB_UPDATE_COMICBOOKS_PUBLISHER = "job.update-comic.publisher";
49-
public static final String JOB_UPDATE_COMICBOOKS_SERIES = "job.update-comic.series";
50-
public static final String JOB_UPDATE_COMICBOOKS_VOLUME = "job.update-comic.volume";
51-
public static final String JOB_UPDATE_COMICBOOKS_ISSUENO = "job.update-comic.issue-number";
52-
public static final String JOB_UPDATE_COMICBOOKS_IMPRINT = "job.update-comic.imprint";
53-
public static final String JOB_UPDATE_COMICBOOKS_COMIC_TYPE = "job.update-comic.comic-type";
47+
public static final String UPDATE_COMIC_BOOKS_JOB = "updateComicBooksJob";
48+
public static final String UPDATE_COMIC_BOOKS_JOB_TIME_STARTED = "job.update-comic.time-started";
49+
public static final String UPDATE_COMIC_BOOKS_JOB_PUBLISHER = "job.update-comic.publisher";
50+
public static final String UPDATE_COMIC_BOOKS_JOB_SERIES = "job.update-comic.series";
51+
public static final String UPDATE_COMIC_BOOKS_JOB_VOLUME = "job.update-comic.volume";
52+
public static final String UPDATE_COMIC_BOOKS_JOB_ISSUE_NUMBER = "job.update-comic.issue-number";
53+
public static final String UPDATE_COMIC_BOOKS_JOB_IMPRINT = "job.update-comic.imprint";
54+
public static final String UPDATE_COMIC_BOOKS_JOB_COMIC_TYPE = "job.update-comic.comic-type";
5455

5556
@Value("${comixed.batch.update-comic-metadata.chunk-size:1}")
5657
private int chunkSize;
5758

58-
@Bean(name = "updateComicBooksJob")
59+
@Bean(name = UPDATE_COMIC_BOOKS_JOB)
5960
public Job updateComicBooksJob(
6061
final JobRepository jobRepository,
6162
@Qualifier("updateComicBooksStep") final Step updateComicBooksStep) {
62-
return new JobBuilder("updateComicBooksJob", jobRepository)
63+
return new JobBuilder(UPDATE_COMIC_BOOKS_JOB, jobRepository)
6364
.incrementer(new RunIdIncrementer())
6465
.start(updateComicBooksStep)
6566
.build();

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/UpdateMetadataConfiguration.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@
4646
@Configuration
4747
@Log4j2
4848
public class UpdateMetadataConfiguration {
49-
public static final String JOB_UPDATE_METADATA_STARTED = "job.update-metadata.started";
5049
public static final String UPDATE_METADATA_JOB = "updateMetadataJob";
50+
public static final String UPDATE_METADATA_JOB_TIME_STARTED = "job.update-metadata.time-started";
5151

5252
@Value("${comixed.batch.update-metadata.chunk-size:10}")
5353
private int chunkSize;

‎comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/listeners/ScrapeMetadataChunkListener.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* ComiXed - A digital comic book library management application.
3+
* Copyright (C) 2024, The ComiXed Project
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses>
17+
*/
18+
19+
package org.comixedproject.batch.comicbooks.listeners;
20+
21+
import static org.comixedproject.batch.comicbooks.ScrapeMetadataConfiguration.SCRAPE_METADATA_STEP;
22+
23+
import lombok.extern.log4j.Log4j2;
24+
import org.springframework.batch.core.configuration.annotation.StepScope;
25+
import org.springframework.stereotype.Component;
26+
27+
@Component
28+
@StepScope
29+
@Log4j2
30+
public class ScrapeMetadataChunkListener extends AbstractBatchProcessChunkListener {
31+
@Override
32+
protected String getStepName() {
33+
return SCRAPE_METADATA_STEP;
34+
}
35+
36+
@Override
37+
protected boolean isActive() {
38+
return this.comicBookService.getBatchScrapingCount() > 0;
39+
}
40+
41+
@Override
42+
protected long getProcessedElements() {
43+
return this.comicBookService.getComicBookCount()
44+
- this.comicBookService.getBatchScrapingCount();
45+
}
46+
47+
@Override
48+
protected long getTotalElements() {
49+
return this.comicBookService.getComicBookCount();
50+
}
51+
}

comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/listeners/ImportComicFilesJobListener.java renamed to comixed-batch/src/main/java/org/comixedproject/batch/comicbooks/listeners/ScrapeMetadataJobListener.java

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818

1919
package org.comixedproject.batch.comicbooks.listeners;
2020

21-
import static org.comixedproject.model.messaging.batch.ProcessComicBooksStatus.*;
22-
2321
import lombok.extern.log4j.Log4j2;
2422
import org.comixedproject.batch.listeners.AbstractBatchProcessListener;
2523
import org.comixedproject.model.batch.BatchProcessDetail;
@@ -29,25 +27,23 @@
2927
import org.springframework.stereotype.Component;
3028

3129
/**
32-
* <code>ImportComicFilesJobListener</code> provides a {@link JobExecutionListener} for the overall
33-
* process of adding comics to the library.
30+
* <code>ScrapeMetadataJobListener</code> defines a listener that publishes status updates while
31+
* running the scrape metadata batch process.
3432
*
3533
* @author Darryl L. Pierce
3634
*/
3735
@Component
3836
@JobScope
3937
@Log4j2
40-
public class ImportComicFilesJobListener extends AbstractBatchProcessListener
38+
public class ScrapeMetadataJobListener extends AbstractBatchProcessListener
4139
implements JobExecutionListener {
4240
@Override
4341
public void beforeJob(final JobExecution jobExecution) {
44-
log.trace("Publishing status before job");
4542
this.doPublishBatchProcessDetail(BatchProcessDetail.from(jobExecution));
4643
}
4744

4845
@Override
4946
public void afterJob(final JobExecution jobExecution) {
50-
log.trace("Publishing status after job");
5147
this.doPublishBatchProcessDetail(BatchProcessDetail.from(jobExecution));
5248
}
5349
}

0 commit comments

Comments
 (0)