Skip to content

Commit 4b6ca73

Browse files
mcpiercemcpierce
committed
Changed add image to cache process to not run in parallel [#2317]
1 parent 4d026c7 commit 4b6ca73

File tree

9 files changed

+120
-39
lines changed

9 files changed

+120
-39
lines changed

comixed-batch/src/main/java/org/comixedproject/batch/comicpages/AddImageCacheEntriesConfiguration.java renamed to comixed-batch/src/main/java/org/comixedproject/batch/comicpages/AddPagesToImageCacheConfiguration.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,15 @@
3636
import org.springframework.transaction.PlatformTransactionManager;
3737

3838
/**
39-
* <code>AddImageCacheEntriesConfiguration</code> defines a batch process that periodically creates
39+
* <code>AddPagesToImageCacheConfiguration</code> defines a batch process that periodically creates
4040
* image cache entries for comic pages that do not have one.
4141
*
4242
* @author Darryl L. Pierce
4343
*/
4444
@Configuration
4545
@Log4j2
46-
public class AddImageCacheEntriesConfiguration {
46+
public class AddPagesToImageCacheConfiguration {
47+
public static final String ADD_PAGES_TO_IMAGE_CACHE_JOB = "addPagesToImageCacheJob";
4748
public static final String PARAM_ADD_IMAGE_CACHE_ENTRIES_STARTED =
4849
"job.add-image-cache-entries.start";
4950

@@ -57,11 +58,11 @@ public class AddImageCacheEntriesConfiguration {
5758
* @param createImageCacheEntriesStep the generate thumbnail step
5859
* @return the job
5960
*/
60-
@Bean(name = "addPageToImageCacheJob")
61-
public Job addPageToImageCacheJob(
61+
@Bean(name = ADD_PAGES_TO_IMAGE_CACHE_JOB)
62+
public Job addPagesToImageCacheJob(
6263
final JobRepository jobRepository,
6364
@Qualifier("createImageCacheEntriesStep") final Step createImageCacheEntriesStep) {
64-
return new JobBuilder("addPageToImageCacheJob", jobRepository)
65+
return new JobBuilder(ADD_PAGES_TO_IMAGE_CACHE_JOB, jobRepository)
6566
.incrementer(new RunIdIncrementer())
6667
.start(createImageCacheEntriesStep)
6768
.next(createImageCacheEntriesStep)

‎comixed-batch/src/main/java/org/comixedproject/batch/comicpages/LoadPageHashesConfiguration.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@
4646
@Configuration
4747
@Log4j2
4848
public class LoadPageHashesConfiguration {
49-
public static final String JOB_LOAD_PAGE_HASHES_STARTED = "job.load-page-hashes.started";
5049
public static final String LOAD_PAGE_HASHES_JOB = "loadPageHashesJob";
50+
public static final String JOB_LOAD_PAGE_HASHES_STARTED = "job.load-page-hashes.started";
5151

5252
@Value("${comixed.batch.load-page-hashes.chunk-size:10}")
5353
private int chunkSize;

comixed-batch/src/main/java/org/comixedproject/batch/initiators/CreateCoverPageImageCacheEntriesInitiator.java renamed to comixed-batch/src/main/java/org/comixedproject/batch/initiators/AddPagesToImageCacheInitiator.java

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919
package org.comixedproject.batch.initiators;
2020

2121
import lombok.extern.log4j.Log4j2;
22-
import org.comixedproject.batch.comicpages.AddImageCacheEntriesConfiguration;
22+
import org.comixedproject.batch.comicpages.AddPagesToImageCacheConfiguration;
23+
import org.comixedproject.service.batch.BatchProcessesService;
24+
import org.comixedproject.service.comicpages.ComicPageService;
2325
import org.comixedproject.service.comicpages.PageCacheService;
2426
import org.springframework.batch.core.Job;
2527
import org.springframework.batch.core.JobParametersBuilder;
@@ -34,19 +36,21 @@
3436
import org.springframework.stereotype.Component;
3537

3638
/**
37-
* <code>CreateCoverPageImageCacheEntriesInitiator</code> provides the entry point to initiate the
38-
* create image cache entries batch process.
39+
* <code>AddPagesToImageCacheInitiator</code> provides the entry point to initiate the add pages to
40+
* the image cache batch process.
3941
*
4042
* @author Darryl L. Pierce
4143
*/
4244
@Component
4345
@Log4j2
44-
public class CreateCoverPageImageCacheEntriesInitiator {
46+
public class AddPagesToImageCacheInitiator {
4547
@Autowired private PageCacheService pageCacheService;
48+
private ComicPageService comicPageService;
49+
@Autowired private BatchProcessesService batchProcessesService;
4650

4751
@Autowired
48-
@Qualifier("addPageToImageCacheJob")
49-
private Job addPageToImageCacheJob;
52+
@Qualifier(AddPagesToImageCacheConfiguration.ADD_PAGES_TO_IMAGE_CACHE_JOB)
53+
private Job addPagesToImageCacheJob;
5054

5155
@Autowired
5256
@Qualifier("batchJobLauncher")
@@ -55,16 +59,21 @@ public class CreateCoverPageImageCacheEntriesInitiator {
5559
/** Starts a batch process to add pages to the image cache. */
5660
@Scheduled(cron = "${comixed.batch.add-cover-to-image-cache.schedule:0 0 * * * *}")
5761
public void execute() {
58-
log.info("Starting process: add pages to image cache");
5962
try {
63+
log.debug("Preparing to process uncached cover pages");
6064
this.pageCacheService.prepareCoverPagesWithoutCacheEntries();
61-
this.jobLauncher.run(
62-
addPageToImageCacheJob,
63-
new JobParametersBuilder()
64-
.addLong(
65-
AddImageCacheEntriesConfiguration.PARAM_ADD_IMAGE_CACHE_ENTRIES_STARTED,
66-
System.currentTimeMillis())
67-
.toJobParameters());
65+
if (this.comicPageService.findPagesNeedingCacheEntriesCount() > 0L
66+
&& !this.batchProcessesService.hasActiveExecutions(
67+
AddPagesToImageCacheConfiguration.ADD_PAGES_TO_IMAGE_CACHE_JOB)) {
68+
log.debug("Starting process: add pages to image cache");
69+
this.jobLauncher.run(
70+
addPagesToImageCacheJob,
71+
new JobParametersBuilder()
72+
.addLong(
73+
AddPagesToImageCacheConfiguration.PARAM_ADD_IMAGE_CACHE_ENTRIES_STARTED,
74+
System.currentTimeMillis())
75+
.toJobParameters());
76+
}
6877
} catch (JobExecutionAlreadyRunningException
6978
| JobInstanceAlreadyCompleteException
7079
| JobParametersInvalidException

‎comixed-batch/src/main/java/org/comixedproject/batch/initiators/LoadPageHashesInitiator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
import org.springframework.stereotype.Component;
3838

3939
/**
40-
* <code>LoadPageHashesInitiator</code> periodically checks for any pages that does not have a
40+
* <code>LoadPageHashesInitiator</code> periodically checks for any page that does not have a
4141
* defined hash. If any are found, and if the <code>library.blocked-pages-enabled</code> feature is
4242
* enabled, then it spawns a batch job to load those hashes.
4343
*

comixed-batch/src/test/java/org/comixedproject/batch/initiators/CreateCoverPageImageCacheEntriesInitiatorTest.java renamed to comixed-batch/src/test/java/org/comixedproject/batch/initiators/AddPagesToImageCacheInitiatorTest.java

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,12 @@
1818

1919
package org.comixedproject.batch.initiators;
2020

21-
import static org.junit.Assert.*;
21+
import static org.comixedproject.batch.comicpages.AddPagesToImageCacheConfiguration.ADD_PAGES_TO_IMAGE_CACHE_JOB;
22+
import static org.junit.Assert.assertNotNull;
2223

23-
import org.comixedproject.batch.comicpages.AddImageCacheEntriesConfiguration;
24+
import org.comixedproject.batch.comicpages.AddPagesToImageCacheConfiguration;
25+
import org.comixedproject.service.batch.BatchProcessesService;
26+
import org.comixedproject.service.comicpages.ComicPageService;
2427
import org.comixedproject.service.comicpages.PageCacheService;
2528
import org.junit.Before;
2629
import org.junit.Test;
@@ -38,13 +41,16 @@
3841
import org.springframework.beans.factory.annotation.Qualifier;
3942

4043
@RunWith(MockitoJUnitRunner.class)
41-
public class CreateCoverPageImageCacheEntriesInitiatorTest {
42-
@InjectMocks private CreateCoverPageImageCacheEntriesInitiator initiator;
44+
public class AddPagesToImageCacheInitiatorTest {
45+
private static final long TEST_PAGE_COUNT = 27L;
46+
@InjectMocks private AddPagesToImageCacheInitiator initiator;
4347
@Mock private PageCacheService pageCacheService;
48+
@Mock private ComicPageService comicPageService;
49+
@Mock private BatchProcessesService batchProcessesService;
4450

4551
@Mock
46-
@Qualifier("addPageToImageCacheJob")
47-
private Job addPageToImageCacheJob;
52+
@Qualifier(ADD_PAGES_TO_IMAGE_CACHE_JOB)
53+
private Job addPagesToImageCacheJob;
4854

4955
@Mock
5056
@Qualifier("batchJobLauncher")
@@ -62,6 +68,9 @@ public void setUp()
6268
JobRestartException {
6369
Mockito.when(jobLauncher.run(Mockito.any(Job.class), jobParametersArgumentCaptor.capture()))
6470
.thenReturn(jobExecution);
71+
Mockito.when(comicPageService.findPagesNeedingCacheEntriesCount()).thenReturn(TEST_PAGE_COUNT);
72+
Mockito.when(batchProcessesService.hasActiveExecutions(ADD_PAGES_TO_IMAGE_CACHE_JOB))
73+
.thenReturn(false);
6574
}
6675

6776
@Test
@@ -76,14 +85,43 @@ public void testExecute()
7685
assertNotNull(jobParameters);
7786
assertNotNull(
7887
jobParameters.getLong(
79-
AddImageCacheEntriesConfiguration.PARAM_ADD_IMAGE_CACHE_ENTRIES_STARTED));
88+
AddPagesToImageCacheConfiguration.PARAM_ADD_IMAGE_CACHE_ENTRIES_STARTED));
8089

8190
Mockito.verify(pageCacheService, Mockito.times(1)).prepareCoverPagesWithoutCacheEntries();
82-
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPageToImageCacheJob, jobParameters);
91+
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPagesToImageCacheJob, jobParameters);
8392
}
8493

8594
@Test
86-
public void testExecuteJobLauncherThrowsException()
95+
public void testExecute_noPagesNeedCaching()
96+
throws JobInstanceAlreadyCompleteException,
97+
JobExecutionAlreadyRunningException,
98+
JobParametersInvalidException,
99+
JobRestartException {
100+
Mockito.when(comicPageService.findPagesNeedingCacheEntriesCount()).thenReturn(0L);
101+
102+
initiator.execute();
103+
104+
Mockito.verify(pageCacheService, Mockito.times(1)).prepareCoverPagesWithoutCacheEntries();
105+
Mockito.verify(jobLauncher, Mockito.never()).run(Mockito.any(Job.class), Mockito.any());
106+
}
107+
108+
@Test
109+
public void testExecute_existingJobFound()
110+
throws JobInstanceAlreadyCompleteException,
111+
JobExecutionAlreadyRunningException,
112+
JobParametersInvalidException,
113+
JobRestartException {
114+
Mockito.when(batchProcessesService.hasActiveExecutions(ADD_PAGES_TO_IMAGE_CACHE_JOB))
115+
.thenReturn(true);
116+
117+
initiator.execute();
118+
119+
Mockito.verify(pageCacheService, Mockito.times(1)).prepareCoverPagesWithoutCacheEntries();
120+
Mockito.verify(jobLauncher, Mockito.never()).run(Mockito.any(Job.class), Mockito.any());
121+
}
122+
123+
@Test
124+
public void testExecute_jobLauncherThrowsException()
87125
throws JobInstanceAlreadyCompleteException,
88126
JobExecutionAlreadyRunningException,
89127
JobParametersInvalidException,
@@ -97,9 +135,9 @@ public void testExecuteJobLauncherThrowsException()
97135
assertNotNull(jobParameters);
98136
assertNotNull(
99137
jobParameters.getLong(
100-
AddImageCacheEntriesConfiguration.PARAM_ADD_IMAGE_CACHE_ENTRIES_STARTED));
138+
AddPagesToImageCacheConfiguration.PARAM_ADD_IMAGE_CACHE_ENTRIES_STARTED));
101139

102140
Mockito.verify(pageCacheService, Mockito.times(1)).prepareCoverPagesWithoutCacheEntries();
103-
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPageToImageCacheJob, jobParameters);
141+
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPagesToImageCacheJob, jobParameters);
104142
}
105143
}

‎comixed-batch/src/test/java/org/comixedproject/batch/initiators/ProcessComicBooksInitiatorTest.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import static org.comixedproject.batch.comicbooks.ProcessComicBooksConfiguration.PROCESS_COMIC_BOOKS_JOB;
2222
import static org.comixedproject.batch.comicbooks.ProcessComicBooksConfiguration.PROCESS_COMIC_BOOKS_STARTED_JOB;
23+
import static org.comixedproject.batch.comicpages.AddPagesToImageCacheConfiguration.ADD_PAGES_TO_IMAGE_CACHE_JOB;
2324
import static org.junit.Assert.*;
2425

2526
import java.util.ArrayList;
@@ -48,8 +49,8 @@ public class ProcessComicBooksInitiatorTest {
4849
@Mock private ComicBookService comicBookService;
4950
@Mock private BatchProcessesService batchProcessesService;
5051

51-
@Mock(name = "addPageToImageCacheJob")
52-
private Job addPageToImageCacheJob;
52+
@Mock(name = ADD_PAGES_TO_IMAGE_CACHE_JOB)
53+
private Job addPagesToImageCacheJob;
5354

5455
@Mock(name = "batchJobLauncher")
5556
private JobLauncher jobLauncher;
@@ -88,7 +89,7 @@ public void testExecuteWithComicsWithoutContent()
8889
assertNotNull(jobParameters.getLong(PROCESS_COMIC_BOOKS_STARTED_JOB));
8990

9091
Mockito.verify(comicBookService, Mockito.times(1)).getUnprocessedComicBookCount();
91-
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPageToImageCacheJob, jobParameters);
92+
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPagesToImageCacheJob, jobParameters);
9293
}
9394

9495
@Test
@@ -106,7 +107,7 @@ public void testExecuteWithComicsFromListener()
106107
assertNotNull(jobParameters.getLong(PROCESS_COMIC_BOOKS_STARTED_JOB));
107108

108109
Mockito.verify(comicBookService, Mockito.times(1)).getUnprocessedComicBookCount();
109-
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPageToImageCacheJob, jobParameters);
110+
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPagesToImageCacheJob, jobParameters);
110111
}
111112

112113
@Test
@@ -123,7 +124,7 @@ public void testExecuteOneComicFound()
123124
assertNotNull(jobParameters);
124125
assertNotNull(jobParameters.getLong(PROCESS_COMIC_BOOKS_STARTED_JOB));
125126

126-
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPageToImageCacheJob, jobParameters);
127+
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPagesToImageCacheJob, jobParameters);
127128
}
128129

129130
@Test
@@ -139,7 +140,7 @@ public void testExecuteWithFinishedProcessing()
139140
assertNotNull(jobParameters);
140141
assertNotNull(jobParameters.getLong(PROCESS_COMIC_BOOKS_STARTED_JOB));
141142

142-
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPageToImageCacheJob, jobParameters);
143+
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPagesToImageCacheJob, jobParameters);
143144
}
144145

145146
@Test
@@ -171,7 +172,7 @@ public void testExecuteJobLauncherThrowsException()
171172
assertNotNull(jobParameters);
172173
assertFalse(jobParameters.isEmpty());
173174

174-
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPageToImageCacheJob, jobParameters);
175+
Mockito.verify(jobLauncher, Mockito.times(1)).run(addPagesToImageCacheJob, jobParameters);
175176
}
176177

177178
@Test

‎comixed-repositories/src/main/java/org/comixedproject/repositories/comicpages/ComicPageRepository.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,14 @@ public interface ComicPageRepository extends JpaRepository<ComicPage, Long> {
9494
@Query("SELECT p FROM ComicPage p WHERE p.addingToCache = true")
9595
List<ComicPage> findPagesNeedingCacheEntries(Pageable pageable);
9696

97+
/**
98+
* Returns the number of page records with the add to cache column set to true.
99+
*
100+
* @return the count
101+
*/
102+
@Query("SELECT COUNT(p) FROM ComicPage p WHERE p.addingToCache = true")
103+
long findPagesNeedingCacheEntriesCount();
104+
97105
/**
98106
* Marks all pages with a given hash as being added to the image cache.
99107
*

‎comixed-services/src/main/java/org/comixedproject/service/comicpages/ComicPageService.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,17 @@ public List<ComicPage> loadPagesNeedingCacheEntries(final int maxRecords) {
190190
return this.comicPageRepository.findPagesNeedingCacheEntries(PageRequest.of(0, maxRecords));
191191
}
192192

193+
/**
194+
* Returns the number of records for pages that need to have a cache entry generated.
195+
*
196+
* @return the count
197+
*/
198+
@Transactional
199+
public long findPagesNeedingCacheEntriesCount() {
200+
log.debug("Loading pages needing image cache entries");
201+
return this.comicPageRepository.findPagesNeedingCacheEntriesCount();
202+
}
203+
193204
/**
194205
* Marks all pages with a given hash as being in the image cache.
195206
*

‎comixed-services/src/test/java/org/comixedproject/service/comicpages/ComicPageServiceTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public class ComicPageServiceTest {
5252
private static final String TEST_PAGE_FILENAME = "src/test/resources/example.jpg";
5353
private static final int TEST_BATCH_SIZE = 129;
5454
private static final String TEST_COMIC_FILENAME = "example-comic.cbz";
55+
private static final long TEST_PAGE_COUNT = 273L;
5556

5657
@InjectMocks private ComicPageService service;
5758
@Mock private ComicPageRepository comicPageRepository;
@@ -259,6 +260,18 @@ public void testLoadPagesNeedingCacheEntries() {
259260
Mockito.verify(comicPageRepository, Mockito.times(1)).findPagesNeedingCacheEntries(pageable);
260261
}
261262

263+
@Test
264+
public void testLoadPagesNeedingCacheEntriesCount() {
265+
Mockito.when(comicPageRepository.findPagesNeedingCacheEntriesCount())
266+
.thenReturn(TEST_PAGE_COUNT);
267+
268+
final long result = service.findPagesNeedingCacheEntriesCount();
269+
270+
assertEquals(TEST_PAGE_COUNT, result);
271+
272+
Mockito.verify(comicPageRepository, Mockito.times(1)).findPagesNeedingCacheEntriesCount();
273+
}
274+
262275
@Test
263276
public void testMarkPagesAsHavingCacheEntry() {
264277
service.markPagesAsHavingCacheEntry(TEST_PAGE_HASH);

0 commit comments

Comments
 (0)