diff --git a/.kotlin/sessions/kotlin-compiler-2216708898994798099.salive b/.kotlin/sessions/kotlin-compiler-2216708898994798099.salive new file mode 100644 index 0000000..e69de29 diff --git a/src/main/kotlin/com/rak/config/AbstractModelDefinition.kt b/src/main/kotlin/com/rak/config/AbstractModelDefinition.kt deleted file mode 100644 index 45c93ed..0000000 --- a/src/main/kotlin/com/rak/config/AbstractModelDefinition.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.rak.config - -import io.smallrye.config.WithName - -interface AbstractModelDefinition { - @WithName("root") - fun rootSelector(): Step -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/CardDefinition.kt b/src/main/kotlin/com/rak/config/CardDefinition.kt deleted file mode 100644 index e199537..0000000 --- a/src/main/kotlin/com/rak/config/CardDefinition.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.rak.config - -import io.smallrye.config.WithName - -interface CardDefinition { - @WithName("name") - fun nameSelector(): SelectorDefinition - @WithName("attack") - fun attackSelector(): SelectorDefinition - @WithName("effect") - fun effectSelector(): SelectorDefinition -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/Items.kt b/src/main/kotlin/com/rak/config/Items.kt deleted file mode 100644 index 31dadb6..0000000 --- a/src/main/kotlin/com/rak/config/Items.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.rak.config - -import java.util.* - -interface Items { - fun card(): Optional - fun regionalSet(): Optional -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/RegionalSetDefinition.kt b/src/main/kotlin/com/rak/config/RegionalSetDefinition.kt deleted file mode 100644 index f5c7b92..0000000 --- a/src/main/kotlin/com/rak/config/RegionalSetDefinition.kt +++ /dev/null @@ -1,12 +0,0 @@ -package com.rak.config - -import io.smallrye.config.WithName - -interface RegionalSetDefinition : AbstractModelDefinition { - @WithName("id") - fun idSelector(): SelectorDefinition - @WithName("language") - fun languageSelector(): SelectorDefinition - @WithName("region-key") - fun regionKeySelector(): SelectorDefinition -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/SelectorDefinition.kt b/src/main/kotlin/com/rak/config/SelectorDefinition.kt deleted file mode 100644 index afaf44f..0000000 --- a/src/main/kotlin/com/rak/config/SelectorDefinition.kt +++ /dev/null @@ -1,8 +0,0 @@ -package com.rak.config - -import java.util.* - -interface SelectorDefinition { - fun steps(): Set - fun transform(): Optional> -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/converter/TypeSelectorConverter.kt b/src/main/kotlin/com/rak/config/converter/TypeSelectorConverter.kt index 4bf2fcf..7c7ff37 100644 --- a/src/main/kotlin/com/rak/config/converter/TypeSelectorConverter.kt +++ b/src/main/kotlin/com/rak/config/converter/TypeSelectorConverter.kt @@ -1,6 +1,6 @@ package com.rak.config.converter -import com.rak.model.scrape.selector.Selector +import com.rak.model.Selector import org.eclipse.microprofile.config.spi.Converter class TypeSelectorConverter : Converter { diff --git a/src/main/kotlin/com/rak/config/model/AbstractScrapeTargetConfig.kt b/src/main/kotlin/com/rak/config/model/AbstractScrapeTargetConfig.kt new file mode 100644 index 0000000..7bcd79c --- /dev/null +++ b/src/main/kotlin/com/rak/config/model/AbstractScrapeTargetConfig.kt @@ -0,0 +1,11 @@ +package com.rak.config.model + +import io.smallrye.config.WithName +import java.util.Optional + +interface AbstractScrapeTargetConfig { + @WithName("root") + fun getRootConfig(): Optional + @WithName("multi") + fun isMulti(): Optional +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/model/CardScrapeTargetConfig.kt b/src/main/kotlin/com/rak/config/model/CardScrapeTargetConfig.kt new file mode 100644 index 0000000..ce983fb --- /dev/null +++ b/src/main/kotlin/com/rak/config/model/CardScrapeTargetConfig.kt @@ -0,0 +1,16 @@ +package com.rak.config.model + +import io.smallrye.config.WithName + +interface CardScrapeTargetConfig : AbstractScrapeTargetConfig { + @WithName("name") + fun getEnglishName(): ScrapeTargetFieldConfig + @WithName("description") + fun getDescription(): ScrapeTargetFieldConfig + @WithName("type") + fun getCardType(): ScrapeTargetFieldConfig + @WithName("attack") + fun getAttack(): ScrapeTargetFieldConfig + @WithName("defense") + fun getDefense(): ScrapeTargetFieldConfig +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/Step.kt b/src/main/kotlin/com/rak/config/model/ExtractConfig.kt similarity index 53% rename from src/main/kotlin/com/rak/config/Step.kt rename to src/main/kotlin/com/rak/config/model/ExtractConfig.kt index f0ceea8..00ff635 100644 --- a/src/main/kotlin/com/rak/config/Step.kt +++ b/src/main/kotlin/com/rak/config/model/ExtractConfig.kt @@ -1,13 +1,14 @@ -package com.rak.config +package com.rak.config.model import com.rak.config.converter.TypeSelectorConverter -import com.rak.model.scrape.selector.Selector +import com.rak.model.Selector import io.smallrye.config.WithConverter import io.smallrye.config.WithName -interface Step { +interface ExtractConfig { @WithConverter(TypeSelectorConverter::class) @WithName("type") - fun selectorType(): Selector // e.g. css or xpath - fun value(): String + fun selectorType(): Selector + @WithName("value") + fun getQueryString(): String } \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/model/RegionalSetScrapeTargetConfig.kt b/src/main/kotlin/com/rak/config/model/RegionalSetScrapeTargetConfig.kt new file mode 100644 index 0000000..c7f22c6 --- /dev/null +++ b/src/main/kotlin/com/rak/config/model/RegionalSetScrapeTargetConfig.kt @@ -0,0 +1,12 @@ +package com.rak.config.model + +import io.smallrye.config.WithName + +interface RegionalSetScrapeTargetConfig : AbstractScrapeTargetConfig { + @WithName("id") + fun idSelector(): ScrapeTargetFieldConfig + @WithName("language") + fun languageSelector(): ScrapeTargetFieldConfig + @WithName("region-key") + fun regionKeySelector(): ScrapeTargetFieldConfig +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/model/ScrapeTargetFieldConfig.kt b/src/main/kotlin/com/rak/config/model/ScrapeTargetFieldConfig.kt new file mode 100644 index 0000000..b7389df --- /dev/null +++ b/src/main/kotlin/com/rak/config/model/ScrapeTargetFieldConfig.kt @@ -0,0 +1,12 @@ +package com.rak.config.model + +import io.smallrye.config.WithName +import java.util.* + +interface ScrapeTargetFieldConfig { + @WithName("root") + fun getRootConfig(): Optional + @WithName("steps") + fun getSteps(): List + fun transform(): Optional> +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/SourceConfig.kt b/src/main/kotlin/com/rak/config/model/SourceConfig.kt similarity index 77% rename from src/main/kotlin/com/rak/config/SourceConfig.kt rename to src/main/kotlin/com/rak/config/model/SourceConfig.kt index c210a2c..06e63db 100644 --- a/src/main/kotlin/com/rak/config/SourceConfig.kt +++ b/src/main/kotlin/com/rak/config/model/SourceConfig.kt @@ -1,4 +1,4 @@ -package com.rak.config +package com.rak.config.model import io.smallrye.config.WithName import java.util.* @@ -13,7 +13,7 @@ interface SourceConfig { fun getDomain(): String @WithName("url-patterns") fun getUrlPatterns(): Optional> - @WithName("selectors") - fun getItems(): Items + @WithName("targets") + fun getTargets(): TargetsConfig } \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/SourcesConfiguration.kt b/src/main/kotlin/com/rak/config/model/SourcesConfig.kt similarity index 77% rename from src/main/kotlin/com/rak/config/SourcesConfiguration.kt rename to src/main/kotlin/com/rak/config/model/SourcesConfig.kt index fb2abe7..dd828ad 100644 --- a/src/main/kotlin/com/rak/config/SourcesConfiguration.kt +++ b/src/main/kotlin/com/rak/config/model/SourcesConfig.kt @@ -1,10 +1,10 @@ -package com.rak.config +package com.rak.config.model import io.smallrye.config.ConfigMapping import io.smallrye.config.WithName @ConfigMapping(prefix = "scraper") -interface SourcesConfiguration { +interface SourcesConfig { @WithName("sources") fun getSources(): MutableList diff --git a/src/main/kotlin/com/rak/config/model/TargetsConfig.kt b/src/main/kotlin/com/rak/config/model/TargetsConfig.kt new file mode 100644 index 0000000..41a884d --- /dev/null +++ b/src/main/kotlin/com/rak/config/model/TargetsConfig.kt @@ -0,0 +1,8 @@ +package com.rak.config.model + +import java.util.* + +interface TargetsConfig { + fun card(): Optional + fun regionalSet(): Optional +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/TransformationStep.kt b/src/main/kotlin/com/rak/config/model/TransformationStepConfig.kt similarity index 76% rename from src/main/kotlin/com/rak/config/TransformationStep.kt rename to src/main/kotlin/com/rak/config/model/TransformationStepConfig.kt index 940e29a..1ad9321 100644 --- a/src/main/kotlin/com/rak/config/TransformationStep.kt +++ b/src/main/kotlin/com/rak/config/model/TransformationStepConfig.kt @@ -1,9 +1,9 @@ -package com.rak.config +package com.rak.config.model import com.rak.config.converter.EmptyStringConverter import io.smallrye.config.WithConverter -interface TransformationStep { +interface TransformationStepConfig { fun name(): String @WithConverter(EmptyStringConverter::class) fun parameters(): MutableList diff --git a/src/main/kotlin/com/rak/controller/ScrapeController.kt b/src/main/kotlin/com/rak/controller/ScrapeController.kt index 69e8930..8ef7f0f 100644 --- a/src/main/kotlin/com/rak/controller/ScrapeController.kt +++ b/src/main/kotlin/com/rak/controller/ScrapeController.kt @@ -1,6 +1,7 @@ package com.rak.controller -import com.rak.model.RegionalSet +import com.rak.model.card.Card +import com.rak.model.set.CardSet import com.rak.service.ScrapeService import jakarta.ws.rs.Consumes import jakarta.ws.rs.GET @@ -27,7 +28,7 @@ class ScrapeController( provider: String, @RestQuery setName: String - ): List { + ): CardSet { return scrapeService.scrapeSet( provider, setName @@ -43,7 +44,7 @@ class ScrapeController( provider: String, @RestQuery cardName: String - ): Map { + ): Card? { return scrapeService.scrapeCard( provider, cardName diff --git a/src/main/kotlin/com/rak/model/RegionalSet.kt b/src/main/kotlin/com/rak/model/RegionalSet.kt deleted file mode 100644 index 6c9bc9e..0000000 --- a/src/main/kotlin/com/rak/model/RegionalSet.kt +++ /dev/null @@ -1,7 +0,0 @@ -package com.rak.model - -data class RegionalSet( - val id: String, - val language: String, - val key: String -) \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/scrape/selector/Selector.kt b/src/main/kotlin/com/rak/model/Selector.kt similarity index 53% rename from src/main/kotlin/com/rak/model/scrape/selector/Selector.kt rename to src/main/kotlin/com/rak/model/Selector.kt index 5a11624..c0eb5b6 100644 --- a/src/main/kotlin/com/rak/model/scrape/selector/Selector.kt +++ b/src/main/kotlin/com/rak/model/Selector.kt @@ -1,4 +1,4 @@ -package com.rak.model.scrape.selector +package com.rak.model enum class Selector { CSS, diff --git a/src/main/kotlin/com/rak/model/card/Attribute.kt b/src/main/kotlin/com/rak/model/card/Attribute.kt new file mode 100644 index 0000000..411ea5f --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/Attribute.kt @@ -0,0 +1,11 @@ +package com.rak.model.card + +enum class Attribute { + WIND, + WATER, + FIRE, + EARTH, + LIGHT, + DARK, + DIVINE; +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/Card.kt b/src/main/kotlin/com/rak/model/card/Card.kt new file mode 100644 index 0000000..0c43642 --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/Card.kt @@ -0,0 +1,8 @@ +package com.rak.model.card + +abstract class Card { + abstract val id: Int + abstract val cardType: CardType + abstract val description: String + abstract val name: String +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/CardType.kt b/src/main/kotlin/com/rak/model/card/CardType.kt new file mode 100644 index 0000000..c6254b5 --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/CardType.kt @@ -0,0 +1,8 @@ +package com.rak.model.card + +enum class CardType { + MONSTER, + SPELL, + TRAP, + UNKNOWN +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/ICardType.kt b/src/main/kotlin/com/rak/model/card/ICardType.kt new file mode 100644 index 0000000..7726896 --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/ICardType.kt @@ -0,0 +1,3 @@ +package com.rak.model.card + +interface ICardType \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/LinkArrow.kt b/src/main/kotlin/com/rak/model/card/LinkArrow.kt new file mode 100644 index 0000000..34dd0af --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/LinkArrow.kt @@ -0,0 +1,12 @@ +package com.rak.model.card + +enum class LinkArrow { + TOP_LEFT, + TOP, + TOP_RIGHT, + LEFT, + RIGHT, + BOTTOM_LEFT, + BOTTOM, + BOTTOM_RIGHT; +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/MonsterCard.kt b/src/main/kotlin/com/rak/model/card/MonsterCard.kt new file mode 100644 index 0000000..a817dcc --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/MonsterCard.kt @@ -0,0 +1,20 @@ +package com.rak.model.card + +data class MonsterCard( + override val id: Int, + override val cardType: CardType, + override val description: String, + override val name: String, + val monsterEffect: String? = null, + val attack: Int? = null, + val defense: Int? = null, + val level: Int? = null, + val isPendulum: Boolean = false, + val pendulumScale: Int? = null, + val pendulumEffect: String? = null, + val linkValue: Int? = null, + val subType: MonsterCardType, + val monsterType: MonsterType, + val attribute: Attribute, + val linkArrows: Set +) : Card() \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/MonsterCardType.kt b/src/main/kotlin/com/rak/model/card/MonsterCardType.kt new file mode 100644 index 0000000..aab8b08 --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/MonsterCardType.kt @@ -0,0 +1,11 @@ +package com.rak.model.card + +enum class MonsterCardType : ICardType { + NORMAL, + EFFECT, + RITUAL, + FUSION, + SYNCHRO, + XYZ, + LINK +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/MonsterType.kt b/src/main/kotlin/com/rak/model/card/MonsterType.kt new file mode 100644 index 0000000..7a977d0 --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/MonsterType.kt @@ -0,0 +1,32 @@ +package com.rak.model.card + +// TODO string value for proper names +// TODO consider adding unknown type +enum class MonsterType { + AQUA, + BEAST, + BEAST_WARRIOR, + CREATOR_GOD, + CYBERSE, + DINOSAUR, + DIVINE_BEAST, + DRAGON, + FAIRY, + FIEND, + FISH, + INSECT, + ILLUSION, + MACHINE, + PLANT, + PSYCHIC, + PYRO, + REPTILE, + ROCK, + SEA_SERPENT, + SPELLCASTER, + THUNDER, + WARRIOR, + WINGED_BEAST, + WYRM, + ZOMBIE +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/SpellCard.kt b/src/main/kotlin/com/rak/model/card/SpellCard.kt new file mode 100644 index 0000000..8879b87 --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/SpellCard.kt @@ -0,0 +1,9 @@ +package com.rak.model.card + +data class SpellCard( + override val id: Int, + override val cardType: CardType, + override val description: String, + override val name: String, + val subType: SpellCardType +) : Card() \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/SpellCardType.kt b/src/main/kotlin/com/rak/model/card/SpellCardType.kt new file mode 100644 index 0000000..27a954c --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/SpellCardType.kt @@ -0,0 +1,11 @@ +package com.rak.model.card + +// TODO fix underscore for all types with string value +enum class SpellCardType { + NORMAL, + CONTINUOUS, + EQUIP, + QUICK_PLAY, + FIELD, + RITUAL +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/card/TrapCard.kt b/src/main/kotlin/com/rak/model/card/TrapCard.kt new file mode 100644 index 0000000..3acdd1f --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/TrapCard.kt @@ -0,0 +1,9 @@ +package com.rak.model.card + +data class TrapCard( + override val id: Int, + override val cardType: CardType, + override val description: String, + override val name: String, + val subType: TrapCardType +) : Card() diff --git a/src/main/kotlin/com/rak/model/card/TrapCardType.kt b/src/main/kotlin/com/rak/model/card/TrapCardType.kt new file mode 100644 index 0000000..04f0fc0 --- /dev/null +++ b/src/main/kotlin/com/rak/model/card/TrapCardType.kt @@ -0,0 +1,7 @@ +package com.rak.model.card + +enum class TrapCardType { + NORMAL, + CONTINUOUS, + COUNTER +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/exception/ElementNotFoundException.kt b/src/main/kotlin/com/rak/model/exception/ElementNotFoundException.kt new file mode 100644 index 0000000..5c39562 --- /dev/null +++ b/src/main/kotlin/com/rak/model/exception/ElementNotFoundException.kt @@ -0,0 +1,3 @@ +package com.rak.model.exception + +class ElementNotFoundException(message: String) : RuntimeException(message) \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/exception/InvalidConfigurationException.kt b/src/main/kotlin/com/rak/model/exception/InvalidConfigurationException.kt new file mode 100644 index 0000000..ddd7688 --- /dev/null +++ b/src/main/kotlin/com/rak/model/exception/InvalidConfigurationException.kt @@ -0,0 +1,3 @@ +package com.rak.model.exception + +class InvalidConfigurationException(message: String) : RuntimeException(message) \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/exception/UnsupportedQueryForProviderException.kt b/src/main/kotlin/com/rak/model/exception/UnsupportedQueryForProviderException.kt new file mode 100644 index 0000000..d24ce93 --- /dev/null +++ b/src/main/kotlin/com/rak/model/exception/UnsupportedQueryForProviderException.kt @@ -0,0 +1,7 @@ +package com.rak.model.exception + +import java.lang.RuntimeException + +class UnsupportedQueryForProviderException( + message: String, +) : RuntimeException(message) \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/scrape/AbstractScraper.kt b/src/main/kotlin/com/rak/model/scrape/AbstractScraper.kt deleted file mode 100644 index 9f4fc12..0000000 --- a/src/main/kotlin/com/rak/model/scrape/AbstractScraper.kt +++ /dev/null @@ -1,4 +0,0 @@ -package com.rak.model.scrape - -abstract class AbstractScraper{ -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/scrape/JsoupScraper.kt b/src/main/kotlin/com/rak/model/scrape/JsoupScraper.kt deleted file mode 100644 index 9cf4df6..0000000 --- a/src/main/kotlin/com/rak/model/scrape/JsoupScraper.kt +++ /dev/null @@ -1,6 +0,0 @@ -package com.rak.model.scrape - -class JsoupScraper : AbstractScraper() { - - -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/scrape/ScrapeJob.kt b/src/main/kotlin/com/rak/model/scrape/ScrapeJob.kt deleted file mode 100644 index 51aa56f..0000000 --- a/src/main/kotlin/com/rak/model/scrape/ScrapeJob.kt +++ /dev/null @@ -1,5 +0,0 @@ -package com.rak.model.scrape - -data class ScrapeJob( - val url: String, -) \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/set/CardSet.kt b/src/main/kotlin/com/rak/model/set/CardSet.kt new file mode 100644 index 0000000..a63c1dc --- /dev/null +++ b/src/main/kotlin/com/rak/model/set/CardSet.kt @@ -0,0 +1,8 @@ +package com.rak.model.set + +import kotlin.collections.Set + +data class CardSet( + val name: String, + val regionalSets: Set +) \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/set/RegionalSet.kt b/src/main/kotlin/com/rak/model/set/RegionalSet.kt new file mode 100644 index 0000000..683325e --- /dev/null +++ b/src/main/kotlin/com/rak/model/set/RegionalSet.kt @@ -0,0 +1,33 @@ +package com.rak.model.set + +data class RegionalSet( + val prefix: String, + val region: String, + val regionCode: String +) { + + companion object { + + fun flattenFromMemberLists( + idList: List, + languageList: List, + regionKeyAliasList: List, + ): MutableSet { + if (idList.size != languageList.size && idList.size != regionKeyAliasList.size) { + throw IllegalArgumentException("Lists have to be the same size") + } + + val regionalSetList: MutableSet = mutableSetOf() + for (index in 0..idList.size - 1) { + regionalSetList.add(RegionalSet( + prefix = idList[index], + region = languageList[index], + regionCode = regionKeyAliasList[index] + )) + } + return regionalSetList + } + + } + +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt b/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt index ad1ecc6..b23bc9c 100644 --- a/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt +++ b/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt @@ -1,6 +1,6 @@ package com.rak.model.transform -import com.rak.config.TransformationStep +import com.rak.config.model.TransformationStepConfig import java.util.concurrent.ConcurrentHashMap class TransformationRegistry { @@ -34,7 +34,7 @@ class TransformationRegistry { parameterizedTransformation.put(name, transformation) } - fun getTransformation(transformationStep: TransformationStep): AbstractTransformation { + fun getTransformation(transformationStep: TransformationStepConfig): AbstractTransformation { val name = transformationStep.name() val parameters = transformationStep.parameters() return when { @@ -56,7 +56,7 @@ class TransformationRegistry { } } - fun applyTransformations(input: String, steps: List): String { + fun applyTransformations(input: String, steps: List): String { return steps.fold(input) { current, step -> val actualStep = getTransformation(step) when (actualStep) { diff --git a/src/main/kotlin/com/rak/service/ExtractionService.kt b/src/main/kotlin/com/rak/service/ExtractionService.kt index 275427d..42d0044 100644 --- a/src/main/kotlin/com/rak/service/ExtractionService.kt +++ b/src/main/kotlin/com/rak/service/ExtractionService.kt @@ -1,44 +1,82 @@ package com.rak.service -import com.rak.config.Step -import com.rak.model.RegionalSet -import com.rak.model.transform.TransformationRegistry +import com.rak.config.model.ExtractConfig +import com.rak.model.Selector +import com.rak.model.card.Card +import com.rak.model.exception.ElementNotFoundException +import com.rak.model.set.CardSet +import com.rak.model.set.RegionalSet import com.rak.util.XPathUtil import jakarta.enterprise.context.ApplicationScoped import org.jsoup.nodes.Document import org.jsoup.nodes.Element +import org.jsoup.select.Elements @ApplicationScoped class ExtractionService( private val sourceService: SourceService, ) { - private val transformationRegistry: TransformationRegistry = TransformationRegistry() + fun extractSet(setName: String, root: Element, provider: String): CardSet { + return CardSet( + name = setName, + regionalSets = extractRegionalSets(root, provider) + ) + } - fun extractSet(document: Document, provider: String): List { + fun extractRegionalSet(root: Element, provider: String): RegionalSet { val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") - val regionalSetSelector = source.getItems().regionalSet().get() + val regionalSetSelector = source.getTargets().regionalSet().get() - val regionalSetRoot = document.select(regionalSetSelector.rootSelector().value()) - - return regionalSetRoot.map { - var setId: String = extractTextFromRootBySteps( - it, - regionalSetSelector.idSelector().steps() + if (regionalSetSelector.getRootConfig().isPresent) { + val setId: String = extractTextFromElementBySteps( + root, + regionalSetSelector.idSelector().getSteps() ) ?: throw IllegalStateException("Parameter 'id' could not be found") - - setId = transformationRegistry.applyTransformations(setId, regionalSetSelector.idSelector().transform().get()) - - val setLanguage: String = extractTextFromRootBySteps( - it, - regionalSetSelector.languageSelector().steps() + val setLanguage: String = extractTextFromElementBySteps( + root, + regionalSetSelector.languageSelector().getSteps() ) ?: throw IllegalStateException("Parameter 'language' could not be found") - val setKey: String = extractTextFromRootBySteps( - it, - regionalSetSelector.regionKeySelector().steps() + val setKey: String = extractTextFromElementBySteps( + root, + regionalSetSelector.regionKeySelector().getSteps() ) ?: throw IllegalStateException("Parameter 'key' could not be found") - RegionalSet( + return RegionalSet( + setId, + setLanguage, + setKey + ) + } else { + val setIdConfiguration = regionalSetSelector.idSelector() + if (!setIdConfiguration.getRootConfig().isPresent) { + throw RuntimeException("as[po") // TODO fix me + } + val rootConfiguration = setIdConfiguration.getRootConfig().get() + + val setIdRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this") + val setId: String = extractTextFromElementBySteps( + setIdRoot, + setIdConfiguration.getSteps() + ) ?: throw IllegalStateException("Parameter 'id' could not be found") + + + val setLanguageConfiguration = regionalSetSelector.idSelector() + val setLanguageRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this") + val setLanguage: String = extractTextFromElementBySteps( + setLanguageRoot, + setLanguageConfiguration.getSteps() + ) ?: throw IllegalStateException("Parameter 'language' could not be found") + + + val setKeyConfiguration = regionalSetSelector.idSelector() + val setKeyRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this") + val setKey: String = extractTextFromElementBySteps( + setKeyRoot, + setKeyConfiguration.getSteps() + ) ?: throw IllegalStateException("Parameter 'key' could not be found") + + return RegionalSet( setId, setLanguage, setKey @@ -46,10 +84,127 @@ class ExtractionService( } } - private fun extractTextFromRootBySteps( + fun extractRegionalSets(root: Element, provider: String): Set { + val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") + val regionalSetSelector = source.getTargets().regionalSet().get() + + if (regionalSetSelector.getRootConfig().isPresent) { + val rootConfiguration = regionalSetSelector.getRootConfig().get() + val regionalSetRoots: Elements = getElementsFromDocumentByExtractConfig( + root, + rootConfiguration + ) + + return regionalSetRoots.map { + extractRegionalSet( + it, + provider + ) + }.toSet() + } else { + val setIdConfiguration = regionalSetSelector.idSelector() + + try { + val setIdRoot = getElementsFromDocumentByExtractConfig(root, setIdConfiguration.getRootConfig().get()) + val setIds = setIdRoot.map { + extractTextFromElementBySteps( + it, + setIdConfiguration.getSteps() + ) ?: throw IllegalStateException("Parameter 'id' could not be found") + } + + val languageConfiguration = regionalSetSelector.languageSelector() + val languageRoot = getElementsFromDocumentByExtractConfig(root, languageConfiguration.getRootConfig().get()) + val languages = languageRoot.map { + extractTextFromElementBySteps( + it, + languageConfiguration.getSteps() + ) ?: throw IllegalStateException("Parameter 'id' could not be found") + } + + val setKeyConfiguration = regionalSetSelector.regionKeySelector() + val setKeyRoot = getElementsFromDocumentByExtractConfig(root, setKeyConfiguration.getRootConfig().get()) + val setKeys = setKeyRoot.map { + extractTextFromElementBySteps( + it, + setKeyConfiguration.getSteps() + ) ?: throw IllegalStateException("Parameter 'id' could not be found") + } + + return RegionalSet.flattenFromMemberLists( + setIds, + languages, + setKeys + ) + } catch (ex: NoSuchElementException) { + throw RuntimeException("sdfgs") // TODO handle me + } + } + } + + fun extractCard(root: Document, provider: String): Card? { + val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") + val cardSelector = source.getTargets().card().get() + + val rootConfigurationOptional = cardSelector.getRootConfig() + + if (rootConfigurationOptional.isPresent) { + val rootConfiguration = rootConfigurationOptional.get() + val rootElement: Element = getElementFromDocumentByExtractConfig( + root, + rootConfiguration + ) ?: throw ElementNotFoundException("TODO make this better") + + val englishCardName: String = extractTextFromElementBySteps( + rootElement, + cardSelector.getEnglishName().getSteps() + ) ?: throw IllegalStateException("Parameter 'name' could not be found") + + val cardType: String = extractTextFromElementBySteps( + rootElement, + cardSelector.getEnglishName().getSteps() + ) ?: throw IllegalStateException("Parameter 'name' could not be found") + + val description: String = extractTextFromElementBySteps( + rootElement, + cardSelector.getEnglishName().getSteps() + ) ?: throw IllegalStateException("Parameter 'name' could not be found") + + return null + } else { + return null + + } + + } + + private fun getElementsFromDocumentByExtractConfig( + document: Element, + step: ExtractConfig + ): Elements { + return if (step.selectorType() == Selector.CSS) { + document.select(step.getQueryString()) + } else { + document.selectXpath(step.getQueryString()) + } + } + + private fun getElementFromDocumentByExtractConfig( + document: Element, + step: ExtractConfig + ): Element? { + return if (step.selectorType() == Selector.CSS) { + document.select(step.getQueryString()).firstOrNull() ?: throw ElementNotFoundException("") + } else { + document.selectXpath(step.getQueryString()).firstOrNull() ?: throw ElementNotFoundException("") + } + } + + private fun extractTextFromElementBySteps( root: Element, - steps: Set + steps: List ): String? { + val stepsInCorrectOrder = steps.reversed() var currentElement: Element? = root.clone() var result: String? = null @@ -60,10 +215,10 @@ class ExtractionService( } if (index == steps.size - 1) { - result = XPathUtil.extractResult(currentElement, currentStep.value()) + result = XPathUtil.extractResult(currentElement, currentStep.getQueryString()) } else { - currentElement = XPathUtil.getNextElement(currentElement, currentStep.value()) + currentElement = XPathUtil.getNextElement(currentElement, currentStep.getQueryString()) } } diff --git a/src/main/kotlin/com/rak/service/ScrapeService.kt b/src/main/kotlin/com/rak/service/ScrapeService.kt index 3611e57..73d6936 100644 --- a/src/main/kotlin/com/rak/service/ScrapeService.kt +++ b/src/main/kotlin/com/rak/service/ScrapeService.kt @@ -1,6 +1,7 @@ package com.rak.service -import com.rak.model.RegionalSet +import com.rak.model.card.Card +import com.rak.model.set.CardSet import jakarta.enterprise.context.ApplicationScoped import org.jsoup.Jsoup import org.jsoup.nodes.Document @@ -15,21 +16,25 @@ class ScrapeService( fun scrapeSet( provider: String, setName: String, - ): List { + ): CardSet { val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") val path: String = normalizePath(setName) val document: Document = Jsoup.connect("https://${source.getDomain()}/$path").get() - return extractionService.extractSet(document, provider) + return extractionService.extractSet(setName, document, provider) } fun scrapeCard( provider: String, cardName: String, - ): Map { + ): Card? { + val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") + val path: String = normalizePath(cardName) - return mapOf() + val document: Document = Jsoup.connect("https://${source.getDomain()}/$path").get() + + return extractionService.extractCard(document, provider) } private fun normalizePath(path: String): String = path diff --git a/src/main/kotlin/com/rak/service/SourceService.kt b/src/main/kotlin/com/rak/service/SourceService.kt index d638f14..c332c5e 100644 --- a/src/main/kotlin/com/rak/service/SourceService.kt +++ b/src/main/kotlin/com/rak/service/SourceService.kt @@ -1,14 +1,86 @@ package com.rak.service -import com.rak.config.SourceConfig -import com.rak.config.SourcesConfiguration +import com.rak.config.model.CardScrapeTargetConfig +import com.rak.config.model.RegionalSetScrapeTargetConfig +import com.rak.config.model.SourceConfig +import com.rak.config.model.SourcesConfig +import com.rak.model.exception.InvalidConfigurationException +import io.quarkus.logging.Log +import io.quarkus.runtime.Startup +import jakarta.annotation.PostConstruct import jakarta.enterprise.context.ApplicationScoped +@Startup @ApplicationScoped -class SourceService ( - val sourcesConfiguration: SourcesConfiguration +class SourceService( + val sourcesConfiguration: SourcesConfig ) { + @PostConstruct + fun init() { + sourcesConfiguration.getSources().forEach { validateSource(it) } + } + + private fun validateSource(sourceConfig: SourceConfig) { + val optionalRegionalSetConfig = sourceConfig.getTargets().regionalSet() + val optionalCardConfig = sourceConfig.getTargets().card() + + if (optionalRegionalSetConfig.isPresent) { + validateSetExtractConfig(optionalRegionalSetConfig.get()) + } + + if (optionalCardConfig.isPresent) { + validateCardExtractConfig(optionalCardConfig.get()) + } + } + + private fun validateSetExtractConfig(setExtractConfig: RegionalSetScrapeTargetConfig) { + val selectors = listOf( + setExtractConfig.languageSelector(), + setExtractConfig.idSelector(), + setExtractConfig.regionKeySelector() + ) + + // If global root is present, dedicated roots may not exist + if (setExtractConfig.getRootConfig().isPresent) { + if (selectors.any { it.getRootConfig().isPresent }) { + throw InvalidConfigurationException( + "Dedicated extraction roots cannot be set when a global extraction root is configured" + ) + } + } else { + if (selectors.any { !it.getRootConfig().isPresent }) { + throw InvalidConfigurationException( + "Dedicated extraction roots have to be set when a global extraction root is not configured" + ) + } + } + } + + private fun validateCardExtractConfig(cardScrapeTargetConfig: CardScrapeTargetConfig) { + val selectors = listOf( + cardScrapeTargetConfig.getEnglishName(), + cardScrapeTargetConfig.getDescription(), + cardScrapeTargetConfig.getCardType(), + cardScrapeTargetConfig.getAttack(), + cardScrapeTargetConfig.getDefense(), + ) + + if (cardScrapeTargetConfig.getRootConfig().isPresent) { + if (selectors.any { it.getRootConfig().isPresent }) { + throw InvalidConfigurationException( + "Dedicated extraction roots cannot be set when a global extraction root is configured" + ) + } + } else { + if (selectors.any { !it.getRootConfig().isPresent }) { + throw InvalidConfigurationException( + "Dedicated extraction roots have to be set when a global extraction root is not configured" + ) + } + } + } + fun getSources(): Set = sourcesConfiguration.getSources().toSet() fun getSourceById(id: String): SourceConfig? = getSources().firstOrNull { it.getId() == id } diff --git a/src/main/kotlin/com/rak/service/TransformService.kt b/src/main/kotlin/com/rak/service/TransformService.kt deleted file mode 100644 index 297e810..0000000 --- a/src/main/kotlin/com/rak/service/TransformService.kt +++ /dev/null @@ -1,13 +0,0 @@ -package com.rak.service - -import com.rak.model.transform.TransformationRegistry -import jakarta.enterprise.context.ApplicationScoped - -@ApplicationScoped -class TransformService( - private val transformationRegistry: TransformationRegistry = TransformationRegistry() -) { - - - -} \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 0d322b7..62283c2 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,8 +9,11 @@ scraper: domain: "yugioh-card.com" url-patterns: - "^https://www\\.yugioh-card\\.com/[a-z]{2}/products/.*$" - selectors: + targets: card: + root: + type: css + value: "h3:contains(Prefix(es)) + div > ul:nth-child(1) > li" name: steps: - type: "css" @@ -27,8 +30,9 @@ scraper: domain: "yugioh.fandom.com" url-patterns: - "^https://yugioh\\.fandom\\.com/wiki/.*$" - selectors: + targets: regional-set: + multi: true root: type: css value: "h3:contains(Prefix(es)) + div > ul:nth-child(1) > li" @@ -50,4 +54,40 @@ scraper: region-key: steps: - type: xpath - value: "//li/abbr/text()" \ No newline at end of file + value: "//li/abbr/text()" + card: + name: + root: + type: css + value: ".cardTable" + steps: + - type: "xpath" + value: "./tbody/tr[3]/th/text()" + description: + root: + type: css + value: ".cardTable" + steps: + - type: "xpath" + value: "b:contains(Card descriptions)" + type: + root: + type: css + value: ".cardTable" + steps: + - type: "xpath" + value: "b:contains(Card descriptions)" + attack: + root: + type: css + value: ".cardTable" + steps: + - type: "xpath" + value: "b:contains(Card descriptions)" + defense: + root: + type: css + value: ".cardTable" + steps: + - type: "xpath" + value: "b:contains(Card descriptions)" \ No newline at end of file