rak/dex-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Amend naming schema

Browse Source
This commit is contained in:
Katarina
2025-06-25 14:10:04 +02:00
parent ce64f90a66
commit 72af626e54
7 changed files with 79 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/kotlin/com/rak/config/model/CardScrapeTargetConfig.kt
View File

@@ -4,13 +4,13 @@ import io.smallrye.config.WithName
interface CardScrapeTargetConfig : AbstractScrapeTargetConfig { interface CardScrapeTargetConfig : AbstractScrapeTargetConfig {
@WithName("name") @WithName("name")
fun getEnglishName(): ScrapeTargetFieldConfig fun getEnglishNameConfig(): ScrapeTargetFieldConfig
@WithName("description") @WithName("description")
fun getDescription(): ScrapeTargetFieldConfig fun getDescriptionConfig(): ScrapeTargetFieldConfig
@WithName("type") @WithName("type")
fun getCardType(): ScrapeTargetFieldConfig fun getCardTypeConfig(): ScrapeTargetFieldConfig
@WithName("attack") @WithName("attack")
fun getAttack(): ScrapeTargetFieldConfig fun getAttackConfig(): ScrapeTargetFieldConfig
@WithName("defense") @WithName("defense")
fun getDefense(): ScrapeTargetFieldConfig fun getDefenseConfig(): ScrapeTargetFieldConfig
} }

2
src/main/kotlin/com/rak/config/model/SourceConfig.kt → src/main/kotlin/com/rak/config/model/ProviderConfig.kt
View File

@@ -3,7 +3,7 @@ package com.rak.config.model
import io.smallrye.config.WithName import io.smallrye.config.WithName
import java.util.* import java.util.*
interface SourceConfig { interface ProviderConfig {
@WithName("id") @WithName("id")
fun getId(): String fun getId(): String

6
src/main/kotlin/com/rak/config/model/RegionalSetScrapeTargetConfig.kt
View File

@@ -4,9 +4,9 @@ import io.smallrye.config.WithName
interface RegionalSetScrapeTargetConfig : AbstractScrapeTargetConfig { interface RegionalSetScrapeTargetConfig : AbstractScrapeTargetConfig {
@WithName("id") @WithName("id")
fun idSelector(): ScrapeTargetFieldConfig fun getIdConfig(): ScrapeTargetFieldConfig
@WithName("language") @WithName("language")
fun languageSelector(): ScrapeTargetFieldConfig fun getLanguageConfig(): ScrapeTargetFieldConfig
@WithName("region-key") @WithName("region-key")
fun regionKeySelector(): ScrapeTargetFieldConfig fun getRegionKeyConfig(): ScrapeTargetFieldConfig
} }

5
src/main/kotlin/com/rak/config/model/ScrapeTargetFieldConfig.kt
View File

@@ -7,6 +7,7 @@ interface ScrapeTargetFieldConfig {
@WithName("root") @WithName("root")
fun getRootConfig(): Optional<ExtractConfig> fun getRootConfig(): Optional<ExtractConfig>
@WithName("steps") @WithName("steps")
fun getSteps(): List<ExtractConfig> fun getExtractionSteps(): List<ExtractConfig>
fun transform(): Optional<List<TransformationStepConfig>> @WithName("transform")
fun getOptionalTransformationSteps(): Optional<List<TransformationStepConfig>>
} }

2
src/main/kotlin/com/rak/config/model/SourcesConfig.kt
View File

@@ -7,6 +7,6 @@ import io.smallrye.config.WithName
interface SourcesConfig { interface SourcesConfig {
@WithName("sources") @WithName("sources")
fun getSources(): MutableList<SourceConfig> fun getSources(): MutableList<ProviderConfig>
} }

90
src/main/kotlin/com/rak/service/ExtractionService.kt
View File

@@ -1,11 +1,13 @@
package com.rak.service package com.rak.service
import com.rak.config.model.ExtractConfig import com.rak.config.model.ExtractConfig
import com.rak.config.model.ScrapeTargetFieldConfig
import com.rak.model.Selector import com.rak.model.Selector
import com.rak.model.card.Card import com.rak.model.card.Card
import com.rak.model.exception.ElementNotFoundException import com.rak.model.exception.ElementNotFoundException
import com.rak.model.set.CardSet import com.rak.model.set.CardSet
import com.rak.model.set.RegionalSet import com.rak.model.set.RegionalSet
import com.rak.model.transform.TransformationRegistry
import com.rak.util.XPathUtil import com.rak.util.XPathUtil
import jakarta.enterprise.context.ApplicationScoped import jakarta.enterprise.context.ApplicationScoped
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
@@ -17,6 +19,8 @@ class ExtractionService(
private val sourceService: SourceService, private val sourceService: SourceService,
) { ) {
private val transformationRegistry = TransformationRegistry()
fun extractSet(setName: String, root: Element, provider: String): CardSet { fun extractSet(setName: String, root: Element, provider: String): CardSet {
return CardSet( return CardSet(
name = setName, name = setName,
@@ -29,17 +33,18 @@ class ExtractionService(
val regionalSetSelector = source.getTargets().regionalSet().get() val regionalSetSelector = source.getTargets().regionalSet().get()
if (regionalSetSelector.getRootConfig().isPresent) { if (regionalSetSelector.getRootConfig().isPresent) {
val setId: String = extractTextFromElementBySteps( val setId: String = extractTextFromElementByTargetFieldConfig(
root, root,
regionalSetSelector.idSelector().getSteps() regionalSetSelector.getIdConfig(),
) ?: throw IllegalStateException("Parameter 'id' could not be found") ) ?: throw IllegalStateException("Parameter 'id' could not be found")
val setLanguage: String = extractTextFromElementBySteps( val setLanguage: String = extractTextFromElementByTargetFieldConfig(
root, root,
regionalSetSelector.languageSelector().getSteps() regionalSetSelector.getLanguageConfig()
) ?: throw IllegalStateException("Parameter 'language' could not be found") ) ?: throw IllegalStateException("Parameter 'language' could not be found")
val setKey: String = extractTextFromElementBySteps( val setKey: String = extractTextFromElementByTargetFieldConfig(
root, root,
regionalSetSelector.regionKeySelector().getSteps() regionalSetSelector.getRegionKeyConfig()
) ?: throw IllegalStateException("Parameter 'key' could not be found") ) ?: throw IllegalStateException("Parameter 'key' could not be found")
return RegionalSet( return RegionalSet(
@@ -48,32 +53,32 @@ class ExtractionService(
setKey setKey
) )
} else { } else {
val setIdConfiguration = regionalSetSelector.idSelector() val setIdConfiguration = regionalSetSelector.getIdConfig()
if (!setIdConfiguration.getRootConfig().isPresent) { if (!setIdConfiguration.getRootConfig().isPresent) {
throw RuntimeException("as[po") // TODO fix me throw RuntimeException("as[po") // TODO fix me
} }
val rootConfiguration = setIdConfiguration.getRootConfig().get() val rootConfiguration = setIdConfiguration.getRootConfig().get()
val setIdRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this") val setIdRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this")
val setId: String = extractTextFromElementBySteps( val setId: String = extractTextFromElementByTargetFieldConfig(
setIdRoot, setIdRoot,
setIdConfiguration.getSteps() setIdConfiguration
) ?: throw IllegalStateException("Parameter 'id' could not be found") ) ?: throw IllegalStateException("Parameter 'id' could not be found")
val setLanguageConfiguration = regionalSetSelector.idSelector() val setLanguageConfiguration = regionalSetSelector.getIdConfig()
val setLanguageRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this") val setLanguageRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this")
val setLanguage: String = extractTextFromElementBySteps( val setLanguage: String = extractTextFromElementByTargetFieldConfig(
setLanguageRoot, setLanguageRoot,
setLanguageConfiguration.getSteps() setLanguageConfiguration
) ?: throw IllegalStateException("Parameter 'language' could not be found") ) ?: throw IllegalStateException("Parameter 'language' could not be found")
val setKeyConfiguration = regionalSetSelector.idSelector() val setKeyConfiguration = regionalSetSelector.getIdConfig()
val setKeyRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this") val setKeyRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this")
val setKey: String = extractTextFromElementBySteps( val setKey: String = extractTextFromElementByTargetFieldConfig(
setKeyRoot, setKeyRoot,
setKeyConfiguration.getSteps() setKeyConfiguration
) ?: throw IllegalStateException("Parameter 'key' could not be found") ) ?: throw IllegalStateException("Parameter 'key' could not be found")
return RegionalSet( return RegionalSet(
@@ -102,32 +107,31 @@ class ExtractionService(
) )
}.toSet() }.toSet()
} else { } else {
val setIdConfiguration = regionalSetSelector.idSelector()
try { try {
val setIdConfiguration = regionalSetSelector.getIdConfig()
val setIdRoot = getElementsFromDocumentByExtractConfig(root, setIdConfiguration.getRootConfig().get()) val setIdRoot = getElementsFromDocumentByExtractConfig(root, setIdConfiguration.getRootConfig().get())
val setIds = setIdRoot.map { val setIds = setIdRoot.map {
extractTextFromElementBySteps( extractTextFromElementByTargetFieldConfig(
it, it,
setIdConfiguration.getSteps() setIdConfiguration
) ?: throw IllegalStateException("Parameter 'id' could not be found") ) ?: throw IllegalStateException("Parameter 'id' could not be found")
} }
val languageConfiguration = regionalSetSelector.languageSelector() val languageConfiguration = regionalSetSelector.getLanguageConfig()
val languageRoot = getElementsFromDocumentByExtractConfig(root, languageConfiguration.getRootConfig().get()) val languageRoot = getElementsFromDocumentByExtractConfig(root, languageConfiguration.getRootConfig().get())
val languages = languageRoot.map { val languages = languageRoot.map {
extractTextFromElementBySteps( extractTextFromElementByTargetFieldConfig(
it, it,
languageConfiguration.getSteps() languageConfiguration
) ?: throw IllegalStateException("Parameter 'id' could not be found") ) ?: throw IllegalStateException("Parameter 'id' could not be found")
} }
val setKeyConfiguration = regionalSetSelector.regionKeySelector() val setKeyConfiguration = regionalSetSelector.getRegionKeyConfig()
val setKeyRoot = getElementsFromDocumentByExtractConfig(root, setKeyConfiguration.getRootConfig().get()) val setKeyRoot = getElementsFromDocumentByExtractConfig(root, setKeyConfiguration.getRootConfig().get())
val setKeys = setKeyRoot.map { val setKeys = setKeyRoot.map {
extractTextFromElementBySteps( extractTextFromElementByTargetFieldConfig(
it, it,
setKeyConfiguration.getSteps() setKeyConfiguration
) ?: throw IllegalStateException("Parameter 'id' could not be found") ) ?: throw IllegalStateException("Parameter 'id' could not be found")
} }
@@ -155,19 +159,19 @@ class ExtractionService(
rootConfiguration rootConfiguration
) ?: throw ElementNotFoundException("TODO make this better") ) ?: throw ElementNotFoundException("TODO make this better")
val englishCardName: String = extractTextFromElementBySteps( val englishCardName: String = extractTextFromElementByTargetFieldConfig(
rootElement, rootElement,
cardSelector.getEnglishName().getSteps() cardSelector.getEnglishNameConfig()
) ?: throw IllegalStateException("Parameter 'name' could not be found") ) ?: throw IllegalStateException("Parameter 'name' could not be found")
val cardType: String = extractTextFromElementBySteps( val cardType: String = extractTextFromElementByTargetFieldConfig(
rootElement, rootElement,
cardSelector.getEnglishName().getSteps() cardSelector.getEnglishNameConfig()
) ?: throw IllegalStateException("Parameter 'name' could not be found") ) ?: throw IllegalStateException("Parameter 'name' could not be found")
val description: String = extractTextFromElementBySteps( val description: String = extractTextFromElementByTargetFieldConfig(
rootElement, rootElement,
cardSelector.getEnglishName().getSteps() cardSelector.getEnglishNameConfig()
) ?: throw IllegalStateException("Parameter 'name' could not be found") ) ?: throw IllegalStateException("Parameter 'name' could not be found")
return null return null
@@ -191,7 +195,7 @@ class ExtractionService(
private fun getElementFromDocumentByExtractConfig( private fun getElementFromDocumentByExtractConfig(
document: Element, document: Element,
step: ExtractConfig step: ExtractConfig,
): Element? { ): Element? {
return if (step.selectorType() == Selector.CSS) { return if (step.selectorType() == Selector.CSS) {
document.select(step.getQueryString()).firstOrNull() ?: throw ElementNotFoundException("") document.select(step.getQueryString()).firstOrNull() ?: throw ElementNotFoundException("")
@@ -200,21 +204,23 @@ class ExtractionService(
} }
} }
private fun extractTextFromElementBySteps( private fun extractTextFromElementByTargetFieldConfig(
root: Element, root: Element,
steps: List<ExtractConfig> extractionConfig: ScrapeTargetFieldConfig
): String? { ): String? {
val stepsInCorrectOrder = steps.reversed() val extractionSteps = extractionConfig.getExtractionSteps()
val transformationSteps = extractionConfig.getOptionalTransformationSteps()
var currentElement: Element? = root.clone() var currentElement: Element? = root.clone()
var result: String? = null var result: String? = null
for (index in 0 until steps.size) { for (index in 0 until extractionSteps.size) {
val currentStep = steps.elementAtOrNull(index) ?: return null val currentStep = extractionSteps.elementAtOrNull(index) ?: return null
if (currentElement == null) { if (currentElement == null) {
throw IllegalStateException() throw IllegalStateException()
} }
if (index == steps.size - 1) { if (index == extractionSteps.size - 1) {
result = XPathUtil.extractResult(currentElement, currentStep.getQueryString()) result = XPathUtil.extractResult(currentElement, currentStep.getQueryString())
} }
else { else {
@@ -222,6 +228,14 @@ class ExtractionService(
} }
} }
if (result == null) {
throw ElementNotFoundException("Result could not be extracted")
}
if (transformationSteps.isPresent) {
result = transformationRegistry.applyTransformations(result, transformationSteps.get())
}
return result return result
} }

29
src/main/kotlin/com/rak/service/SourceService.kt
View File

@@ -2,10 +2,9 @@ package com.rak.service
import com.rak.config.model.CardScrapeTargetConfig import com.rak.config.model.CardScrapeTargetConfig
import com.rak.config.model.RegionalSetScrapeTargetConfig import com.rak.config.model.RegionalSetScrapeTargetConfig
import com.rak.config.model.SourceConfig import com.rak.config.model.ProviderConfig
import com.rak.config.model.SourcesConfig import com.rak.config.model.SourcesConfig
import com.rak.model.exception.InvalidConfigurationException import com.rak.model.exception.InvalidConfigurationException
import io.quarkus.logging.Log
import io.quarkus.runtime.Startup import io.quarkus.runtime.Startup
import jakarta.annotation.PostConstruct import jakarta.annotation.PostConstruct
import jakarta.enterprise.context.ApplicationScoped import jakarta.enterprise.context.ApplicationScoped
@@ -21,9 +20,9 @@ class SourceService(
sourcesConfiguration.getSources().forEach { validateSource(it) } sourcesConfiguration.getSources().forEach { validateSource(it) }
} }
private fun validateSource(sourceConfig: SourceConfig) { private fun validateSource(providerConfig: ProviderConfig) {
val optionalRegionalSetConfig = sourceConfig.getTargets().regionalSet() val optionalRegionalSetConfig = providerConfig.getTargets().regionalSet()
val optionalCardConfig = sourceConfig.getTargets().card() val optionalCardConfig = providerConfig.getTargets().card()
if (optionalRegionalSetConfig.isPresent) { if (optionalRegionalSetConfig.isPresent) {
validateSetExtractConfig(optionalRegionalSetConfig.get()) validateSetExtractConfig(optionalRegionalSetConfig.get())
@@ -36,9 +35,9 @@ class SourceService(
private fun validateSetExtractConfig(setExtractConfig: RegionalSetScrapeTargetConfig) { private fun validateSetExtractConfig(setExtractConfig: RegionalSetScrapeTargetConfig) {
val selectors = listOf( val selectors = listOf(
setExtractConfig.languageSelector(), setExtractConfig.getLanguageConfig(),
setExtractConfig.idSelector(), setExtractConfig.getIdConfig(),
setExtractConfig.regionKeySelector() setExtractConfig.getRegionKeyConfig()
) )
// If global root is present, dedicated roots may not exist // If global root is present, dedicated roots may not exist
@@ -59,11 +58,11 @@ class SourceService(
private fun validateCardExtractConfig(cardScrapeTargetConfig: CardScrapeTargetConfig) { private fun validateCardExtractConfig(cardScrapeTargetConfig: CardScrapeTargetConfig) {
val selectors = listOf( val selectors = listOf(
cardScrapeTargetConfig.getEnglishName(), cardScrapeTargetConfig.getEnglishNameConfig(),
cardScrapeTargetConfig.getDescription(), cardScrapeTargetConfig.getDescriptionConfig(),
cardScrapeTargetConfig.getCardType(), cardScrapeTargetConfig.getCardTypeConfig(),
cardScrapeTargetConfig.getAttack(), cardScrapeTargetConfig.getAttackConfig(),
cardScrapeTargetConfig.getDefense(), cardScrapeTargetConfig.getDefenseConfig(),
) )
if (cardScrapeTargetConfig.getRootConfig().isPresent) { if (cardScrapeTargetConfig.getRootConfig().isPresent) {
@@ -81,7 +80,7 @@ class SourceService(
} }
} }
fun getSources(): Set<SourceConfig> = sourcesConfiguration.getSources().toSet() fun getSources(): Set<ProviderConfig> = sourcesConfiguration.getSources().toSet()
fun getSourceById(id: String): SourceConfig? = getSources().firstOrNull { it.getId() == id } fun getSourceById(id: String): ProviderConfig? = getSources().firstOrNull { it.getId() == id }
} }