У меня есть следующий класс, и я использую ForkJoinPool:
Код: Выделить всё
@Service
@RequiredArgsConstructor
public class PageIndexingService {
private final SiteModelRepository sitesRepository;
private final PageRepository pageRepository;
private final LemmaRepository lemmaRepository;
private final IndexRepository indexRepository;
private final AppConfig appConfig;
private final SitesList sitesList;
private final DataCleanupService dataCleanupService;
private final LemmaParserService lemmaParserService;
private final ForkJoinPool forkJoinPool = new ForkJoinPool();
private static final Logger logger = LoggerFactory.getLogger(PageIndexingService.class);
@PreDestroy
public void cleanUp() {
try {
forkJoinPool.shutdown();
if (!forkJoinPool.awaitTermination(15, TimeUnit.SECONDS)) {
forkJoinPool.shutdownNow();
}
} catch (InterruptedException e) {
forkJoinPool.shutdownNow();
Thread.currentThread().interrupt();
}
}
public IndexApiResponse indexPage(String url) {
try {
if (sitesRepository.count() == 0) {
populateDatabaseFromConfig();
}
if (pageRepository.existsBySite(site)) {
dataCleanupService.deleteOldSiteData(site);
}
site.setStatus(Status.INDEXING);
site.setStatusTime(LocalDateTime.now());
sitesRepository.save(site);
indexPages(site, url);
site.setStatus(Status.INDEXED);
sitesRepository.save(site);
return new IndexApiResponse(true, null);
} catch (Exception e) {
return new IndexApiResponse(false, "Indexing error: " + e.getMessage());
}
}
private void populateDatabaseFromConfig() {
//
}
private void indexPages(SiteModel site, String url) {
forkJoinPool.submit(() -> processPage(site, url, new HashSet()));
}
private void processPage(SiteModel site, String url, Set visitedUrls) {
if (visitedUrls.contains(url)) return;
visitedUrls.add(url);
logger.info("Indexing page: " + url + " in thread: " + Thread.currentThread().getName());
try {
PageFetchResult fetchResult = fetchDocumentWithStatus(url);
if (fetchResult.getStatusCode() >= 400) {
logger.error("HTTP error: {}", fetchResult.getStatusCode());
return;
}
Page page = savePage(site, url, fetchResult.getStatusCode(), fetchResult.getDocument().html());
String cleanContent = cleanHtml(fetchResult.getDocument().html());
Map lemmaCounts = parseLemmas(cleanContent);
updateLemmaAndIndex(site, page, cleanContent, lemmaCounts);
processLinks(site, fetchResult.getDocument(), visitedUrls);
} catch (Exception e) {
logger.error("HTML parsing error: {}", url, e);
}
}
private void updateLemmaAndIndex(SiteModel site, Page page, String cleanContent, Map lemmaCounts) {
for (Map.Entry entry : lemmaCounts.entrySet()) {
Lemma lemma = lemmaRepository.findByLemmaAndSite(entry.getKey(), site)
.orElse(new Lemma());
lemma.setSite(site);
lemma.setLemma(entry.getKey());
lemma.setFrequency(lemma.getFrequency() + 1);
lemmaRepository.save(lemma);
IndexModel index = new IndexModel();
index.setPage(page);
index.setLemma(lemma);
index.setRank(entry.getValue() * 1.0f / cleanContent.length());
indexRepository.save(index);
}
}
//other helper methods
}
Код: Выделить всё
2024-12-01 00:59:15.848 WARN 13788 --- [Pool-1-worker-5] o.h.e.loading.internal.LoadContexts : HHH000100: Fail-safe cleanup (collections) : org.hibernate.engine.loading.internal.CollectionLoadContext@451d00e9
2024-12-01 00:59:15.849 WARN 13788 --- [Pool-1-worker-5] o.h.e.l.internal.CollectionLoadContext : HHH000160: On CollectionLoadContext#cleanup, localLoadingCollectionKeys contained [1] entries
Код: Выделить всё
org.springframework.beans.factory.BeanCreationNotAllowedException: Error creating bean with name 'dataSourceScriptDatabaseInitializer': Singleton bean creation not allowed while singletons of this factory are in destruction (Do not request a bean from a BeanFactory in a destroy method implementation!)
Код: Выделить всё
lemmaRepository.save(lemma);
Мой первый код был без метода @PreDestroy (читал об этом и пробовал ), а метод indexPages был:
Код: Выделить всё
private void indexPages(SiteModel site, String url) {
ForkJoinPool forkJoinPool = new ForkJoinPool();
forkJoinPool.submit(() -> processPage(site, url, new HashSet()));
forkJoinPool.shutdown();
}
Подробнее здесь: https://stackoverflow.com/questions/792 ... eading-app