diff --git a/src/main/kotlin/com/rak/config/SelectorDefinition.kt b/src/main/kotlin/com/rak/config/SelectorDefinition.kt index b20b9b0..afaf44f 100644 --- a/src/main/kotlin/com/rak/config/SelectorDefinition.kt +++ b/src/main/kotlin/com/rak/config/SelectorDefinition.kt @@ -4,5 +4,5 @@ import java.util.* interface SelectorDefinition { fun steps(): Set - fun transform(): Optional> + fun transform(): Optional> } \ No newline at end of file diff --git a/src/main/kotlin/com/rak/config/TransformationStep.kt b/src/main/kotlin/com/rak/config/TransformationStep.kt index c0d23a4..940e29a 100644 --- a/src/main/kotlin/com/rak/config/TransformationStep.kt +++ b/src/main/kotlin/com/rak/config/TransformationStep.kt @@ -6,5 +6,5 @@ import io.smallrye.config.WithConverter interface TransformationStep { fun name(): String @WithConverter(EmptyStringConverter::class) - fun parameters(): MutableList + fun parameters(): MutableList } \ No newline at end of file diff --git a/src/main/kotlin/com/rak/controller/ExampleResource.kt b/src/main/kotlin/com/rak/controller/ScrapeController.kt similarity index 69% rename from src/main/kotlin/com/rak/controller/ExampleResource.kt rename to src/main/kotlin/com/rak/controller/ScrapeController.kt index 01cd9a4..69e8930 100644 --- a/src/main/kotlin/com/rak/controller/ExampleResource.kt +++ b/src/main/kotlin/com/rak/controller/ScrapeController.kt @@ -1,8 +1,7 @@ package com.rak.controller -import com.rak.config.SourcesConfiguration +import com.rak.model.RegionalSet import com.rak.service.ScrapeService -import com.rak.service.SourceService import jakarta.ws.rs.Consumes import jakarta.ws.rs.GET import jakarta.ws.rs.Path @@ -13,15 +12,11 @@ import org.jboss.resteasy.reactive.RestQuery @Path("/api") -class ExampleResource( - private val sourcesConfiguration: SourcesConfiguration, +class ScrapeController( private val scrapeService: ScrapeService, - private val sourceService: SourceService ) { - companion object { - private val TEXT_NODE_MATCHER: Regex = Regex("text\\(\\)$") - } + @GET @Path("/{provider}/set") @@ -32,8 +27,8 @@ class ExampleResource( provider: String, @RestQuery setName: String - ): List> { - return scrapeService.extractSet( + ): List { + return scrapeService.scrapeSet( provider, setName ) @@ -49,7 +44,7 @@ class ExampleResource( @RestQuery cardName: String ): Map { - return scrapeService.extractCard( + return scrapeService.scrapeCard( provider, cardName ) diff --git a/src/main/kotlin/com/rak/model/RegionalSet.kt b/src/main/kotlin/com/rak/model/RegionalSet.kt new file mode 100644 index 0000000..6c9bc9e --- /dev/null +++ b/src/main/kotlin/com/rak/model/RegionalSet.kt @@ -0,0 +1,7 @@ +package com.rak.model + +data class RegionalSet( + val id: String, + val language: String, + val key: String +) \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/transform/ParameterizedTransformation.kt b/src/main/kotlin/com/rak/model/transform/ParameterizedTransformation.kt index bca8fd8..3630e2e 100644 --- a/src/main/kotlin/com/rak/model/transform/ParameterizedTransformation.kt +++ b/src/main/kotlin/com/rak/model/transform/ParameterizedTransformation.kt @@ -2,5 +2,5 @@ package com.rak.model.transform @FunctionalInterface fun interface ParameterizedTransformation : AbstractTransformation { - fun apply(input: String, vararg parameters: String): Any? + fun apply(input: String, parameters: List): String } \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/transform/Transformation.kt b/src/main/kotlin/com/rak/model/transform/Transformation.kt index 2b924ee..799c2e4 100644 --- a/src/main/kotlin/com/rak/model/transform/Transformation.kt +++ b/src/main/kotlin/com/rak/model/transform/Transformation.kt @@ -2,5 +2,5 @@ package com.rak.model.transform @FunctionalInterface fun interface Transformation : AbstractTransformation { - fun apply(input: String): String? + fun apply(input: String): String } \ No newline at end of file diff --git a/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt b/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt index 1dfc2c5..ad1ecc6 100644 --- a/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt +++ b/src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt @@ -1,5 +1,6 @@ package com.rak.model.transform +import com.rak.config.TransformationStep import java.util.concurrent.ConcurrentHashMap class TransformationRegistry { @@ -33,17 +34,19 @@ class TransformationRegistry { parameterizedTransformation.put(name, transformation) } - fun getTransformation(name: String, parameters: List? = null): AbstractTransformation { + fun getTransformation(transformationStep: TransformationStep): AbstractTransformation { + val name = transformationStep.name() + val parameters = transformationStep.parameters() return when { transformations.containsKey(name) -> { - if (!parameters.isNullOrEmpty()) { + if (parameters.isNotEmpty()) { throw IllegalArgumentException("'$name' doesn't accept parameters") } else { transformations[name]!! } } parameterizedTransformation.containsKey(name) -> { - if (parameters.isNullOrEmpty()) { + if (parameters.isEmpty()) { throw IllegalArgumentException("'$name' requires parameters") } else { parameterizedTransformation[name]!! @@ -53,4 +56,21 @@ class TransformationRegistry { } } + fun applyTransformations(input: String, steps: List): String { + return steps.fold(input) { current, step -> + val actualStep = getTransformation(step) + when (actualStep) { + is Transformation -> + transformations[step.name()]?.apply(current) + ?: throw IllegalArgumentException("Unknown transformation: ${step.name()}") + + is ParameterizedTransformation -> + parameterizedTransformation[step.name()]?.apply(current, step.parameters()) + ?: throw IllegalArgumentException("Unknown transformation: ${step.name()}") + + else -> throw IllegalStateException("Invalid transformation type") + } + } + } + } \ No newline at end of file diff --git a/src/main/kotlin/com/rak/service/ExtractionService.kt b/src/main/kotlin/com/rak/service/ExtractionService.kt new file mode 100644 index 0000000..275427d --- /dev/null +++ b/src/main/kotlin/com/rak/service/ExtractionService.kt @@ -0,0 +1,73 @@ +package com.rak.service + +import com.rak.config.Step +import com.rak.model.RegionalSet +import com.rak.model.transform.TransformationRegistry +import com.rak.util.XPathUtil +import jakarta.enterprise.context.ApplicationScoped +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element + +@ApplicationScoped +class ExtractionService( + private val sourceService: SourceService, +) { + + private val transformationRegistry: TransformationRegistry = TransformationRegistry() + + fun extractSet(document: Document, provider: String): List { + val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") + val regionalSetSelector = source.getItems().regionalSet().get() + + val regionalSetRoot = document.select(regionalSetSelector.rootSelector().value()) + + return regionalSetRoot.map { + var setId: String = extractTextFromRootBySteps( + it, + regionalSetSelector.idSelector().steps() + ) ?: throw IllegalStateException("Parameter 'id' could not be found") + + setId = transformationRegistry.applyTransformations(setId, regionalSetSelector.idSelector().transform().get()) + + val setLanguage: String = extractTextFromRootBySteps( + it, + regionalSetSelector.languageSelector().steps() + ) ?: throw IllegalStateException("Parameter 'language' could not be found") + val setKey: String = extractTextFromRootBySteps( + it, + regionalSetSelector.regionKeySelector().steps() + ) ?: throw IllegalStateException("Parameter 'key' could not be found") + + RegionalSet( + setId, + setLanguage, + setKey + ) + } + } + + private fun extractTextFromRootBySteps( + root: Element, + steps: Set + ): String? { + var currentElement: Element? = root.clone() + var result: String? = null + + for (index in 0 until steps.size) { + val currentStep = steps.elementAtOrNull(index) ?: return null + if (currentElement == null) { + throw IllegalStateException() + } + + if (index == steps.size - 1) { + result = XPathUtil.extractResult(currentElement, currentStep.value()) + } + else { + currentElement = XPathUtil.getNextElement(currentElement, currentStep.value()) + } + } + + return result + } + +} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/service/MyRemoteService.kt b/src/main/kotlin/com/rak/service/MyRemoteService.kt deleted file mode 100644 index c79aae0..0000000 --- a/src/main/kotlin/com/rak/service/MyRemoteService.kt +++ /dev/null @@ -1,29 +0,0 @@ -package com.rak.service - -import jakarta.ws.rs.GET -import jakarta.ws.rs.Path -import jakarta.ws.rs.QueryParam -import org.eclipse.microprofile.rest.client.inject.RegisterRestClient - -/** - * To use it via injection. - * - * ```kotlin - * @Inject - * @RestClient - * lateinit var myRemoteService: MyRemoteService - * - * fun doSomething() { - * val restClientExtensions = myRemoteService.getExtensionsById("io.quarkus:quarkus-rest-client") - * } - * ``` - */ -@RegisterRestClient(baseUri = "https://stage.code.quarkus.io/api") -interface MyRemoteService { - - @GET - @Path("/extensions") - fun getExtensionsById(@QueryParam("id") id: String): Set - - data class Extension(val id: String, val name: String, val shortName: String, val keywords: List) -} \ No newline at end of file diff --git a/src/main/kotlin/com/rak/service/ScrapeService.kt b/src/main/kotlin/com/rak/service/ScrapeService.kt index c8c5095..3611e57 100644 --- a/src/main/kotlin/com/rak/service/ScrapeService.kt +++ b/src/main/kotlin/com/rak/service/ScrapeService.kt @@ -1,86 +1,30 @@ package com.rak.service -import com.rak.config.RegionalSetDefinition -import com.rak.config.SourcesConfiguration -import com.rak.config.Step -import com.rak.util.XPathUtil +import com.rak.model.RegionalSet import jakarta.enterprise.context.ApplicationScoped import org.jsoup.Jsoup import org.jsoup.nodes.Document -import org.jsoup.nodes.Element @ApplicationScoped class ScrapeService( - private val sourceService: SourceService + private val sourceService: SourceService, + private val extractionService: ExtractionService, ) { - companion object { - private val TEXT_NODE_MATCHER: Regex = Regex("text\\(\\)$") - } - private fun extractTextFromRootBySteps( - root: Element, - steps: Set - ): String? { - var currentElement: Element? = root.clone() - var result: String? = null - - for (index in 0 until steps.size) { - val currentStep = steps.elementAtOrNull(index) ?: return null - if (currentElement == null) { - throw IllegalStateException() - } - - if (index == steps.size - 1) { - result = XPathUtil.extractResult(currentElement, currentStep.value()) - } - else { - currentElement = XPathUtil.getNextElement(currentElement, currentStep.value()) - } - } - - - return result - } - - fun extractSet( + fun scrapeSet( provider: String, setName: String, - ): List> { - val source = - sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") - + ): List { + val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found") val path: String = normalizePath(setName) val document: Document = Jsoup.connect("https://${source.getDomain()}/$path").get() - val regionalSetSelector = source.getItems().regionalSet().get() - val regionalSetRoot = document.select(regionalSetSelector.rootSelector().value()) - - return regionalSetRoot.map { - val setId: String? = extractTextFromRootBySteps( - it, - regionalSetSelector.idSelector().steps() - ) - val setLanguage: String? = extractTextFromRootBySteps( - it, - regionalSetSelector.languageSelector().steps() - ) - val setKey: String? = extractTextFromRootBySteps( - it, - regionalSetSelector.regionKeySelector().steps() - ) - - mapOf( - Pair("id", setId ?: "N/A"), - Pair("language", setLanguage ?: "N/A"), - Pair("key", setKey ?: "N/A"), - ) - } + return extractionService.extractSet(document, provider) } - - fun extractCard( + fun scrapeCard( provider: String, cardName: String, ): Map { diff --git a/src/main/kotlin/com/rak/service/TransformService.kt b/src/main/kotlin/com/rak/service/TransformService.kt new file mode 100644 index 0000000..297e810 --- /dev/null +++ b/src/main/kotlin/com/rak/service/TransformService.kt @@ -0,0 +1,13 @@ +package com.rak.service + +import com.rak.model.transform.TransformationRegistry +import jakarta.enterprise.context.ApplicationScoped + +@ApplicationScoped +class TransformService( + private val transformationRegistry: TransformationRegistry = TransformationRegistry() +) { + + + +} \ No newline at end of file