Bump version

Regards Set ID

.gitea/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Create and Push Release
+on:
+  workflow_dispatch:
+
+env:
+  AUTHENTIK_URL: https://auth.smoothbrain.win
+  REGISTRY_URL: gitea.smoothbrain.win
+  IMAGE_OWNER: rak
+  IMAGE_NAME: dex-scraper-java
+
+jobs:
+
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup JDK
+        uses: https://gitea.smoothbrain.win/rak/setup-java@main
+        with:
+          distribution: 'corretto'
+          java-version: '21.0.6'
+          cache: 'gradle'
+
+      - name: Build & Push Image
+        env:
+          QUARKUS_CONTAINER_IMAGE_USERNAME: ${{ secrets.CI_SERVICE_ACCOUNT }}
+          QUARKUS_CONTAINER_IMAGE_PASSWORD: ${{ secrets.CI_SERVICE_ACCOUNT_PASSWORD }}
+        run: |
+          ./gradlew clean build \
+            -Dquarkus.container-image.push=true

build.gradle.kts

@@ -14,6 +14,7 @@ val quarkusPlatformArtifactId: String by project
 val quarkusPlatformVersion: String by project
 
 dependencies {
+    implementation("io.quarkus:quarkus-container-image-docker")
     implementation("io.quarkus:quarkus-config-yaml")
     implementation(enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}"))
     implementation("io.quarkus:quarkus-rest")
@@ -22,15 +23,19 @@ dependencies {
     implementation("io.quarkus:quarkus-rest-client-kotlin-serialization")
     implementation("io.quarkus:quarkus-rest-jackson")
     implementation("io.quarkus:quarkus-kotlin")
+    implementation("io.quarkus:quarkus-smallrye-fault-tolerance")
     implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8")
-    implementation("org.jsoup:jsoup:1.20.1")
     implementation("io.quarkus:quarkus-arc")
+    implementation("org.jsoup:jsoup:1.20.1")
+    implementation("org.netpreserve.commons:webarchive-commons:2.0.1")
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.19.0")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.19.1")
     testImplementation("io.quarkus:quarkus-junit5")
     testImplementation("io.rest-assured:rest-assured")
 }
 
 group = "com.rak"
-version = "1.0-SNAPSHOT"
+version = "0.0.2"
 
 java {
     sourceCompatibility = JavaVersion.VERSION_21

settings.gradle.kts

@@ -10,4 +10,4 @@ pluginManagement {
         id(quarkusPluginId) version quarkusPluginVersion
     }
 }
-rootProject.name = "jsoup-scraper"
+rootProject.name = "dex-scraper"

sources.yml

@@ -1,47 +0,0 @@
-scraper:
-  sources:
-    - id: konami-official
-      name: "Konami Official Database"
-      domain: "yugioh-card.com"
-      url-patterns:
-        - "^https://www\\.yugioh-card\\.com/[a-z]{2}/products/.*$"
-      selectors:
-        card:
-          name:
-            steps:
-              - type: "css"
-                value: "h1.product-title"
-              - type: "xpath"
-                value: "//h1[@itemprop='name']"
-          attack:
-            steps:
-              - type: "css"
-                value: ".atk-value"
-
-    - id: ygo-fandom
-      name: "Yu-Gi-Oh Fandom Wiki"
-      domain: "yugioh.fandom.com"
-      url-patterns:
-        - "^https://yugioh\\.fandom\\.com/wiki/.*$"
-      selectors:
-        regional-set:
-          root: "h3:contains(Prefix(es)) + div > ul:nth-child(1) > li"
-          id:
-            steps:
-              - type: "xpath"
-                value: "//li/text()"
-            transform:
-              - name: "replace"
-                parameters:
-                  - " ("
-                  - ""
-          language:
-            steps:
-              - type: "xpath"
-                value: "//li/abbr"
-              - type: "xpath"
-                value: "//abbr/@title"
-          region-key:
-            steps:
-              - type: "xpath"
-                value: "//li/abbr/text()"

src/main/kotlin/com/rak/config/converter/EmptyStringConverter.kt

@@ -1,11 +0,0 @@
-package com.rak.config.converter
-
-import jakarta.annotation.Priority
-import org.eclipse.microprofile.config.spi.Converter
-
-@Priority(1)
-class EmptyStringConverter : Converter<String> {
-    override fun convert(value: String): String {
-        return value
-    }
-}

src/main/kotlin/com/rak/config/converter/PatternConverter.kt

@@ -0,0 +1,19 @@
+package com.rak.config.converter
+
+import org.eclipse.microprofile.config.spi.Converter
+import java.util.regex.Pattern
+import java.util.regex.PatternSyntaxException
+
+class PatternConverter : Converter<Pattern> {
+    override fun convert(value: String): Pattern {
+        if (value.isBlank()) {
+            throw IllegalArgumentException("Pattern may not be empty")
+        }
+
+        try {
+            return Pattern.compile(value)
+        } catch (_: PatternSyntaxException) {
+            throw IllegalStateException("'$value' is not a valid RegEx pattern")
+        }
+    }
+}

src/main/kotlin/com/rak/config/model/DiscriminatorConfig.kt

@@ -6,7 +6,4 @@ import io.smallrye.config.WithConverter
 import io.smallrye.config.WithName
 
 interface DiscriminatorConfig : ScrapeTargetFieldConfig {
-    @WithName("direction")
-    @WithConverter(DiscriminatorDirectionConverter::class)
-    fun getDiscriminatorDirection(): DiscriminatorDirection
 }

src/main/kotlin/com/rak/config/model/ExtractorConfig.kt

@@ -0,0 +1,11 @@
+package com.rak.config.model
+
+import io.smallrye.config.WithName
+import java.util.Optional
+
+interface ExtractorConfig {
+    @WithName("steps")
+    fun getExtractionSteps(): List<ExtractConfig>
+    @WithName("transform")
+    fun getOptionalTransformationSteps(): Optional<List<TransformationStepConfig>>
+}

src/main/kotlin/com/rak/config/model/FieldConfigFallback.kt

@@ -0,0 +1,15 @@
+package com.rak.config.model
+
+import io.smallrye.config.WithDefault
+import io.smallrye.config.WithName
+import java.util.Optional
+
+interface FieldConfigFallback {
+    @WithName("steps")
+    fun getOptionalFallbackExtractionSteps(): Optional<List<ExtractConfig>>
+    @WithName("transform")
+    fun getOptionalTransformationSteps(): Optional<List<TransformationStepConfig>>
+    @WithName("default")
+    @WithDefault("N/A")
+    fun getOptionalDefaultValue(): String
+}

src/main/kotlin/com/rak/config/model/ProviderConfig.kt

@@ -11,8 +11,8 @@ interface ProviderConfig {
     fun getName(): String
     @WithName("domain")
     fun getDomain(): String
-    @WithName("url-patterns")
-    fun getUrlPatterns(): Optional<MutableSet<String>>
+    @WithName("url-pattern")
+    fun getUrlPattern(): String
     @WithName("targets")
     fun getTargets(): TargetsConfig
 

src/main/kotlin/com/rak/config/model/RegionalSetScrapeTargetConfig.kt

@@ -0,0 +1,12 @@
+package com.rak.config.model
+
+import io.smallrye.config.WithName
+
+interface RegionalSetScrapeTargetConfig : AbstractScrapeTargetConfig {
+    @WithName("id")
+    fun getIdConfig(): ScrapeTargetFieldConfig
+    @WithName("language")
+    fun getLanguageConfig(): ScrapeTargetFieldConfig
+    @WithName("region-key")
+    fun getRegionKeyConfig(): ScrapeTargetFieldConfig
+}

src/main/kotlin/com/rak/config/model/ScrapeTargetFieldConfig.kt

@@ -1,13 +1,21 @@
 package com.rak.config.model
 
+import io.smallrye.config.WithDefault
 import io.smallrye.config.WithName
 import java.util.*
 
 interface ScrapeTargetFieldConfig : AbstractScrapeTargetFieldConfig {
+    @WithName("type")
+    fun getType(): String
+    @WithName("nullable")
+    @WithDefault("false")
+    fun isNullable(): Boolean
     @WithName("root")
     fun getRootConfig(): Optional<ExtractConfig>
-    @WithName("steps")
-    fun getExtractionSteps(): List<ExtractConfig>
-    @WithName("transform")
-    fun getOptionalTransformationSteps(): Optional<List<TransformationStepConfig>>
+    @WithName("extractors")
+    fun getExtractionMethods(): List<ExtractorConfig>
+    @WithName("fallback")
+    fun getFallbackConfiguration(): Optional<FieldConfigFallback>
+    @WithName("validation")
+    fun getOptionalValidation(): Optional<ValidationConfig>
 }

src/main/kotlin/com/rak/config/model/SetScrapeTargetConfig.kt

@@ -3,10 +3,6 @@ package com.rak.config.model
 import io.smallrye.config.WithName
 
 interface SetScrapeTargetConfig : AbstractScrapeTargetConfig {
-    @WithName("id")
-    fun getIdConfig(): ScrapeTargetFieldConfig
-    @WithName("language")
-    fun getLanguageConfig(): ScrapeTargetFieldConfig
-    @WithName("region-key")
-    fun getRegionKeyConfig(): ScrapeTargetFieldConfig
+    @WithName("name")
+    fun getNameConfig(): ScrapeTargetFieldConfig
 }

src/main/kotlin/com/rak/config/model/TargetsConfig.kt

@@ -8,6 +8,8 @@ interface TargetsConfig {
     fun getCardConfig(): Optional<CardScrapeTargetConfig>
     @WithName("set")
     fun getSetConfig(): Optional<SetScrapeTargetConfig>
+    @WithName("regional-set")
+    fun getRegionalSetConfig(): Optional<RegionalSetScrapeTargetConfig>
     @WithName("card-print")
     fun getCardPrintConfiguration(): Optional<CardPrintScrapeTargetConfig>
 }

src/main/kotlin/com/rak/config/model/TransformationStepConfig.kt

@@ -1,10 +1,8 @@
 package com.rak.config.model
 
-import com.rak.config.converter.EmptyStringConverter
-import io.smallrye.config.WithConverter
+import java.util.Optional
 
 interface TransformationStepConfig {
     fun name(): String
-    @WithConverter(EmptyStringConverter::class)
-    fun parameters(): MutableList<String>
+    fun parameters(): Optional<MutableList<String>>
 }

src/main/kotlin/com/rak/config/model/ValidationConfig.kt

@@ -0,0 +1,12 @@
+package com.rak.config.model
+
+import com.rak.config.converter.PatternConverter
+import io.smallrye.config.WithConverter
+import io.smallrye.config.WithName
+import java.util.regex.Pattern
+
+interface ValidationConfig {
+    @WithName("pattern")
+    @WithConverter(PatternConverter::class)
+    fun getRegexPatterns(): MutableList<Pattern>
+}

src/main/kotlin/com/rak/controller/ScrapeController.kt

@@ -1,8 +1,10 @@
 package com.rak.controller
 
 import com.rak.model.card.Card
+import com.rak.model.cc.CCIndexSuccessResponse
 import com.rak.model.set.CardSet
 import com.rak.model.set.RegionalSet
+import com.rak.service.CommonCrawlService
 import com.rak.service.ScrapeService
 import jakarta.ws.rs.Consumes
 import jakarta.ws.rs.GET
@@ -16,10 +18,9 @@ import org.jboss.resteasy.reactive.RestQuery
 @Path("/api")
 class ScrapeController(
     private val scrapeService: ScrapeService,
+    private val commonCrawlService: CommonCrawlService
 ) {
 
-
-
     @GET
     @Path("/{provider}/set")
     @Produces(MediaType.APPLICATION_JSON)

src/main/kotlin/com/rak/model/ErrorResponse.kt

@@ -0,0 +1,5 @@
+package com.rak.model
+
+data class ErrorResponse(
+    val message: String
+)

src/main/kotlin/com/rak/model/card/CardPrint.kt

@@ -1,9 +1,7 @@
 package com.rak.model.card
 
-import com.rak.model.set.RegionalSet
-
 data class CardPrint(
-    val id: String,
+    var id: Int,
     val name: String,
     val regionalName: String? = null,
     val rarity: String
@@ -11,10 +9,17 @@ data class CardPrint(
 
     companion object {
         fun fromMap(map: Map<String, String>): CardPrint {
+            val regionalNameValue = map["regionalName"]
+            val regionalName = if (regionalNameValue == "") {
+                null
+            } else {
+                regionalNameValue
+            }
+
             return CardPrint(
-                map["id"] ?: throw IllegalStateException("Parameter 'prefix' not found"),
+                map["id"]?.toInt() ?: throw IllegalStateException("Parameter 'prefix' not found"),
                 map["name"] ?: throw IllegalStateException("Parameter 'region' not found"),
-                map["regionalName"],
+                regionalName,
                 map["rarity"] ?: throw IllegalStateException("Parameter 'regionCode' not found"),
             )
         }

src/main/kotlin/com/rak/model/cc/CCIndexErrorResponse.kt

@@ -0,0 +1,5 @@
+package com.rak.model.cc
+
+data class CCIndexErrorResponse(
+    val message: String
+)

src/main/kotlin/com/rak/model/cc/CCIndexSuccessResponse.kt

@@ -0,0 +1,22 @@
+package com.rak.model.cc
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import java.time.Instant
+
+data class CCIndexSuccessResponse(
+    @JsonProperty("urlkey")
+    val urlKey: String,
+    val timestamp: Instant,
+    val url: String,
+    val mime: String,
+    @JsonProperty("mime-detected")
+    val mimeDetected: String,
+    val status: String,
+    val digest: String,
+    val length: Int,
+    val offset: Int,
+    @JsonProperty("filename")
+    val fileName: String,
+    val languages: String,
+    val encoding: String,
+)

src/main/kotlin/com/rak/model/cc/CCIndices.kt

@@ -0,0 +1,9 @@
+package com.rak.model.cc
+
+enum class CCIndices(val indexName: String) {
+    CC_2025_21("CC-MAIN-2025-21"),
+    CC_2025_05("CC-MAIN-2024-05"),
+    CC_2024_46("CC-MAIN-2024-46"),
+    CC_2024_26("CC-MAIN-2024-26"),
+    CC_2023_50("CC-MAIN-2023-50");
+}

src/main/kotlin/com/rak/model/exception/TargetNotFoundException.kt

@@ -0,0 +1,3 @@
+package com.rak.model.exception
+
+class TargetNotFoundException(message: String) : RuntimeException(message)

src/main/kotlin/com/rak/model/exception/ValueValidationException.kt

@@ -0,0 +1,3 @@
+package com.rak.model.exception
+
+class ValueValidationException(message: String) : RuntimeException(message)

src/main/kotlin/com/rak/model/exception/mapper/NotImplementedExceptionMapper.kt

@@ -0,0 +1,18 @@
+package com.rak.model.exception.mapper
+
+import com.rak.model.ErrorResponse
+import com.rak.model.exception.NotImplementedException
+import jakarta.ws.rs.core.Response
+import jakarta.ws.rs.ext.ExceptionMapper
+import jakarta.ws.rs.ext.Provider
+
+@Provider
+class NotImplementedExceptionMapper : ExceptionMapper<NotImplementedException> {
+    override fun toResponse(exception: NotImplementedException): Response {
+        return Response.status(405).entity(
+            ErrorResponse(
+                exception.message ?: "Provider does not implement this method"
+            )
+        ).build()
+    }
+}

src/main/kotlin/com/rak/model/exception/mapper/TargetNotFoundExceptionMapper.kt

@@ -0,0 +1,19 @@
+package com.rak.model.exception.mapper
+
+import com.rak.model.ErrorResponse
+import com.rak.model.exception.NotImplementedException
+import com.rak.model.exception.TargetNotFoundException
+import jakarta.ws.rs.core.Response
+import jakarta.ws.rs.ext.ExceptionMapper
+import jakarta.ws.rs.ext.Provider
+
+@Provider
+class TargetNotFoundExceptionMapper : ExceptionMapper<TargetNotFoundException> {
+    override fun toResponse(exception: TargetNotFoundException): Response {
+        return Response.status(404).entity(
+            ErrorResponse(
+                exception.message ?: "Scrape target could not be found"
+            )
+        ).build()
+    }
+}

src/main/kotlin/com/rak/model/set/CardSet.kt

@@ -1,12 +1,15 @@
 package com.rak.model.set
 
-import kotlin.collections.Set
-
 data class CardSet(
-    val name: String,
+    var name: String,
     val regionalSets: Set<RegionalSet>
 ) {
     companion object {
-
+        fun fromMap(map: Map<String, String>, regionalSet: Set<RegionalSet>): CardSet {
+            return CardSet(
+                map["name"] ?: throw IllegalStateException("Parameter 'name' not found"),
+                regionalSet
+            )
+        }
     }
 }

src/main/kotlin/com/rak/model/set/RegionalSet.kt

@@ -22,28 +22,6 @@ data class RegionalSet(
             )
         }
 
-        fun flattenFromMemberLists(
-            idList: List<String>,
-            languageList: List<String>,
-            regionKeyAliasList: List<String>,
-        ): MutableSet<RegionalSet> {
-            if (idList.size != languageList.size && idList.size != regionKeyAliasList.size) {
-                throw IllegalArgumentException("Lists have to be the same size")
-            }
-
-            val regionalSetList: MutableSet<RegionalSet> = mutableSetOf()
-            for (index in 0..idList.size - 1) {
-                regionalSetList.add(RegionalSet(
-                    prefix = idList[index],
-                    region = languageList[index],
-                    regionCode = regionKeyAliasList[index],
-                    listOf(),
-                    numberOfCards = -1
-                ))
-            }
-            return regionalSetList
-        }
-
     }
 
 }

src/main/kotlin/com/rak/model/transform/ParameterizedTransformation.kt

@@ -2,5 +2,5 @@ package com.rak.model.transform
 
 @FunctionalInterface
 fun interface ParameterizedTransformation : AbstractTransformation {
-    fun apply(input: String, parameters: List<String>): String
+    fun apply(input: String, parameters: MutableList<String>): String
 }

src/main/kotlin/com/rak/model/transform/TransformationRegistry.kt

@@ -11,15 +11,22 @@ class TransformationRegistry {
 
     init {
         register("trim") { it.trim() }
+        register("removeInnerQuotes") { it.replace(Regex("^\""), "").replace(Regex("\"$"), "") }
         register("replace") { input, parameters ->
-            require(parameters.size == 2) {
-                "'replace' requires exactly 2 parameters"
+            require(parameters.size == 1 || parameters.size == 2) {
+                "'replace' requires either 1 or 2 parameters"
+            }
+            if (parameters.size == 1) {
+                parameters.add("")
             }
             input.replace(parameters[0], parameters[1])
         }
         register("regexReplace") { input, params ->
-            require(params.size == 2) {
-                "'regexReplace' requires exactly 2 parameters"
+            require(params.size == 1 || params.size == 2) {
+                "'regexReplace' requires either 1 or 2 parameters"
+            }
+            if (params.size == 1) {
+                params.add("")
             }
             input.replace(params[0].toRegex(), params[1])
         }
@@ -39,14 +46,14 @@ class TransformationRegistry {
         val parameters = transformationStep.parameters()
         return when {
             transformations.containsKey(name) -> {
-                if (parameters.isNotEmpty()) {
+                if (parameters.isPresent && parameters.get().isNotEmpty()) {
                     throw IllegalArgumentException("'$name' doesn't accept parameters")
                 } else {
                     transformations[name]!!
                 }
             }
             parameterizedTransformation.containsKey(name) -> {
-                if (parameters.isEmpty()) {
+                if (parameters.isPresent && parameters.get().isEmpty()) {
                     throw IllegalArgumentException("'$name' requires parameters")
                 } else {
                     parameterizedTransformation[name]!!
@@ -65,7 +72,7 @@ class TransformationRegistry {
                         ?: throw IllegalArgumentException("Unknown transformation: ${step.name()}")
 
                 is ParameterizedTransformation ->
-                    parameterizedTransformation[step.name()]?.apply(current, step.parameters())
+                    parameterizedTransformation[step.name()]?.apply(current, step.parameters().get())
                         ?: throw IllegalArgumentException("Unknown transformation: ${step.name()}")
 
                 else -> throw IllegalStateException("Invalid transformation type")

src/main/kotlin/com/rak/service/CommonCrawlService.kt

@@ -0,0 +1,92 @@
+package com.rak.service
+
+import com.rak.model.cc.CCIndexSuccessResponse
+import com.rak.model.cc.CCIndices
+import com.rak.service.client.CommonCrawlRestClient
+import io.netty.buffer.ByteBufInputStream
+import io.quarkus.logging.Log
+import jakarta.enterprise.context.ApplicationScoped
+import org.archive.format.http.HttpResponseParser
+import org.archive.io.warc.WARCReaderFactory
+import org.eclipse.microprofile.rest.client.inject.RestClient
+import org.jsoup.helper.DataUtil
+import org.jsoup.nodes.Document
+
+
+@ApplicationScoped
+class CommonCrawlService(
+    @RestClient
+    private val commonCrawlRestClient: CommonCrawlRestClient
+) {
+
+    companion object {
+        private const val INDEX_QUERY_URL: String = "http://index.commoncrawl.org"
+        private const val DATA_URL: String = "http://data.commoncrawl.org"
+    }
+
+    fun queryIndex(
+        url: String
+    ): CCIndexSuccessResponse {
+        return commonCrawlRestClient.queryIndex(
+            INDEX_QUERY_URL,
+            url,
+            CCIndices.CC_2024_46.indexName
+        )
+    }
+
+    fun queryAllCrawlIndices(
+        url: String
+    ): List<CCIndexSuccessResponse> {
+        val responses = mutableListOf<CCIndexSuccessResponse>()
+        for (crawlName in CCIndices.entries) {
+            try {
+                responses.add(commonCrawlRestClient.queryIndex(
+                    INDEX_QUERY_URL,
+                    url,
+                    crawlName.indexName
+                ))
+            } catch (ex: RuntimeException) {
+                Log.warn("Error occurred querying crawl '${crawlName.indexName}' for URL $url", ex)
+            }
+        }
+
+        return responses
+    }
+
+    fun getDocument(
+        ccIndexSuccessResponse: CCIndexSuccessResponse,
+        baseUri: String
+    ): Document? {
+        val fileName = "CC-MAIN-20241106230027-20241107020027-00740.warc.gz"
+        val buf: ByteBufInputStream = commonCrawlRestClient.getWarcArchive(
+            DATA_URL,
+            ccIndexSuccessResponse.fileName,
+            ccIndexSuccessResponse.length,
+            ccIndexSuccessResponse.offset
+        )
+
+        val test = WARCReaderFactory.get(
+            fileName,
+            buf,
+            true
+        )
+
+        val parser = HttpResponseParser()
+
+        for(record in test) {
+            val http = parser.parse(record.buffered())
+            val charSet = http.headers.get("charset")
+
+            val doc = DataUtil.load(
+                http.buffered(),
+                "UTF-8",
+                baseUri
+            )
+
+            return doc
+        }
+
+        return null
+    }
+
+}

src/main/kotlin/com/rak/service/ExtractionService.kt

@@ -1,273 +0,0 @@
-package com.rak.service
-
-import com.rak.config.model.CardPrintScrapeTargetConfig
-import com.rak.config.model.ExtractConfig
-import com.rak.config.model.ScrapeTargetFieldConfig
-import com.rak.model.Selector
-import com.rak.model.card.Card
-import com.rak.model.card.CardPrint
-import com.rak.model.exception.ElementNotFoundException
-import com.rak.model.exception.InvalidConfigurationException
-import com.rak.model.set.CardSet
-import com.rak.model.set.RegionalSet
-import com.rak.model.transform.TransformationRegistry
-import com.rak.util.XPathUtil
-import jakarta.enterprise.context.ApplicationScoped
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
-import org.jsoup.select.Elements
-import java.util.Optional
-
-@ApplicationScoped
-class ExtractionService(
-    private val sourceService: SourceService,
-) {
-
-    private val transformationRegistry = TransformationRegistry()
-
-    fun extractSet(setName: String, root: Element, provider: String): CardSet {
-        return CardSet(
-            name = setName,
-            regionalSets = extractRegionalSets(root, provider)
-        )
-    }
-
-    fun getRootElement(
-        document: Document,
-        globalRootExtractConfig: Optional<ExtractConfig>,
-        nodeRootExtractConfig: Optional<ExtractConfig>
-    ): Element {
-        val rootExtractConfig: ExtractConfig = globalRootExtractConfig.orElse(
-            nodeRootExtractConfig.orElseThrow {
-                InvalidConfigurationException("")
-            })
-
-        return getElementFromDocumentByExtractConfig(document, rootExtractConfig) ?: throw ElementNotFoundException("No root could be found")
-    }
-
-    fun extractCardPrint(document: Document, cardPrintConfig: CardPrintScrapeTargetConfig): CardPrint? {
-        val cardName = extractTextFromElementByTargetFieldConfig(
-            getRootElement(
-                document,
-                cardPrintConfig.getRootConfig(),
-                cardPrintConfig.getNameConfig().getRootConfig()
-            ),
-            cardPrintConfig.getNameConfig()
-        )
-
-        return null
-    }
-
-    fun extractRegionalSet(root: Element, provider: String): RegionalSet {
-        val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found")
-        val setExtractionConfig = source.getTargets().getSetConfig().get()
-
-        if (setExtractionConfig.getRootConfig().isPresent) {
-            val setId: String = extractTextFromElementByTargetFieldConfig(
-                root,
-                setExtractionConfig.getIdConfig(),
-
-            ) ?: throw IllegalStateException("Parameter 'id' could not be found")
-            val setLanguage: String = extractTextFromElementByTargetFieldConfig(
-                root,
-                setExtractionConfig.getLanguageConfig()
-            ) ?: throw IllegalStateException("Parameter 'language' could not be found")
-            val setKey: String = extractTextFromElementByTargetFieldConfig(
-                root,
-                setExtractionConfig.getRegionKeyConfig()
-            ) ?: throw IllegalStateException("Parameter 'key' could not be found")
-
-            return RegionalSet(
-                setId,
-                setLanguage,
-                setKey,
-                listOf(),
-                -1
-            )
-        } else {
-            val setIdConfiguration = setExtractionConfig.getIdConfig()
-            val rootConfiguration = setIdConfiguration.getRootConfig().get()
-
-            val setIdRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this")
-            val setId: String = extractTextFromElementByTargetFieldConfig(
-                setIdRoot,
-                setIdConfiguration
-            ) ?: throw IllegalStateException("Parameter 'id' could not be found")
-
-
-            val setLanguageConfiguration = setExtractionConfig.getIdConfig()
-            val setLanguageRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this")
-            val setLanguage: String = extractTextFromElementByTargetFieldConfig(
-                setLanguageRoot,
-                setLanguageConfiguration
-            ) ?: throw IllegalStateException("Parameter 'language' could not be found")
-
-
-            val setKeyConfiguration = setExtractionConfig.getIdConfig()
-            val setKeyRoot = getElementFromDocumentByExtractConfig(root, rootConfiguration) ?: throw ElementNotFoundException("TODO fix this")
-            val setKey: String = extractTextFromElementByTargetFieldConfig(
-                setKeyRoot,
-                setKeyConfiguration
-            ) ?: throw IllegalStateException("Parameter 'key' could not be found")
-
-            return RegionalSet(
-                setId,
-                setLanguage,
-                setKey,
-                listOf(),
-                -1
-            )
-        }
-    }
-
-    fun extractRegionalSets(root: Element, provider: String): Set<RegionalSet> {
-        val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found")
-        val setExtractionConfig = source.getTargets().getSetConfig().get()
-
-        if (setExtractionConfig.getRootConfig().isPresent) {
-            val rootConfiguration = setExtractionConfig.getRootConfig().get()
-            val regionalSetRoots: Elements = getElementsFromDocumentByExtractConfig(
-                root,
-                rootConfiguration
-            )
-
-            return regionalSetRoots.map {
-                extractRegionalSet(
-                    it,
-                    provider
-                )
-            }.toSet()
-        } else {
-            try {
-                val setIdConfiguration = setExtractionConfig.getIdConfig()
-                val setIdRoot = getElementsFromDocumentByExtractConfig(root, setIdConfiguration.getRootConfig().get())
-                val setIds = setIdRoot.map {
-                    extractTextFromElementByTargetFieldConfig(
-                        it,
-                        setIdConfiguration
-                    ) ?: throw IllegalStateException("Parameter 'id' could not be found")
-                }
-
-                val languageConfiguration = setExtractionConfig.getLanguageConfig()
-                val languageRoot = getElementsFromDocumentByExtractConfig(root, languageConfiguration.getRootConfig().get())
-                val languages = languageRoot.map {
-                    extractTextFromElementByTargetFieldConfig(
-                        it,
-                        languageConfiguration
-                    ) ?: throw IllegalStateException("Parameter 'id' could not be found")
-                }
-
-                val setKeyConfiguration = setExtractionConfig.getRegionKeyConfig()
-                val setKeyRoot = getElementsFromDocumentByExtractConfig(root, setKeyConfiguration.getRootConfig().get())
-                val setKeys = setKeyRoot.map {
-                    extractTextFromElementByTargetFieldConfig(
-                        it,
-                        setKeyConfiguration
-                    ) ?: throw IllegalStateException("Parameter 'id' could not be found")
-                }
-
-                return RegionalSet.flattenFromMemberLists(
-                    setIds,
-                    languages,
-                    setKeys
-                )
-            } catch (ex: NoSuchElementException) {
-                throw RuntimeException("sdfgs") // TODO handle me
-            }
-        }
-    }
-
-    fun extractCard(root: Document, provider: String): Card? {
-        val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found")
-        val cardSelector = source.getTargets().getCardConfig().get()
-
-        val rootConfigurationOptional = cardSelector.getRootConfig()
-
-        if (rootConfigurationOptional.isPresent) {
-            val rootConfiguration = rootConfigurationOptional.get()
-            val rootElement: Element = getElementFromDocumentByExtractConfig(
-                root,
-                rootConfiguration
-            ) ?: throw ElementNotFoundException("TODO make this better")
-
-            val englishCardName: String = extractTextFromElementByTargetFieldConfig(
-                rootElement,
-                cardSelector.getEnglishNameConfig()
-            ) ?: throw IllegalStateException("Parameter 'name' could not be found")
-
-            val cardType: String = extractTextFromElementByTargetFieldConfig(
-                rootElement,
-                cardSelector.getEnglishNameConfig()
-            ) ?: throw IllegalStateException("Parameter 'name' could not be found")
-
-            val description: String = extractTextFromElementByTargetFieldConfig(
-                rootElement,
-                cardSelector.getEnglishNameConfig()
-            ) ?: throw IllegalStateException("Parameter 'name' could not be found")
-
-            return null
-        } else {
-            return null
-
-        }
-
-    }
-
-    private fun getElementsFromDocumentByExtractConfig(
-        document: Element,
-        step: ExtractConfig
-    ): Elements {
-        return if (step.selectorType() == Selector.CSS) {
-            document.select(step.getQueryString())
-        } else {
-            document.selectXpath(step.getQueryString())
-        }
-    }
-
-    private fun getElementFromDocumentByExtractConfig(
-        document: Element,
-        step: ExtractConfig,
-    ): Element? {
-        return if (step.selectorType() == Selector.CSS) {
-            document.select(step.getQueryString()).firstOrNull() ?: throw ElementNotFoundException("")
-        } else {
-            document.selectXpath(step.getQueryString()).firstOrNull() ?: throw ElementNotFoundException("")
-        }
-    }
-
-    private fun extractTextFromElementByTargetFieldConfig(
-        root: Element,
-        extractionConfig: ScrapeTargetFieldConfig
-    ): String? {
-        val extractionSteps = extractionConfig.getExtractionSteps()
-        val transformationSteps = extractionConfig.getOptionalTransformationSteps()
-
-        var currentElement: Element? = root.clone()
-        var result: String? = null
-
-        for (index in 0 until extractionSteps.size) {
-            val currentStep = extractionSteps.elementAtOrNull(index) ?: return null
-            if (currentElement == null) {
-                throw IllegalStateException()
-            }
-
-            if (index == extractionSteps.size - 1) {
-                result = XPathUtil.extractResult(currentElement, currentStep.getQueryString())
-            }
-            else {
-                currentElement = XPathUtil.getNextElement(currentElement, currentStep.getQueryString())
-            }
-        }
-
-        if (result == null) {
-            throw ElementNotFoundException("Result could not be extracted")
-        }
-
-        if (transformationSteps.isPresent) {
-            result = transformationRegistry.applyTransformations(result, transformationSteps.get())
-        }
-
-        return result
-    }
-
-}

src/main/kotlin/com/rak/service/RegionalSetExtractionService.kt

@@ -1,52 +0,0 @@
-package com.rak.service
-
-import com.rak.config.model.ProviderConfig
-import com.rak.config.model.ScrapeTargetFieldConfig
-import com.rak.config.model.SetScrapeTargetConfig
-import com.rak.config.model.SourcesConfig
-import com.rak.model.exception.NotImplementedException
-import com.rak.model.set.RegionalSet
-import jakarta.enterprise.context.ApplicationScoped
-import org.jsoup.nodes.Element
-
-@ApplicationScoped
-class RegionalSetExtractionService(
-    private val cardPrintExtractionService: CardPrintExtractionService,
-    private val sourcesConfig: SourcesConfig
-) : AbstractExtractionService<RegionalSet, SetScrapeTargetConfig>() {
-
-    override fun SetScrapeTargetConfig.getItems(): Map<String, ScrapeTargetFieldConfig> {
-        return mapOf(
-            Pair("prefix", this.getIdConfig()),
-            Pair("regionCode", this.getRegionKeyConfig()),
-            Pair("region", this.getLanguageConfig()),
-        )
-    }
-
-    override fun extract(
-        element: Element,
-        providerConfig: ProviderConfig,
-        extractionConfig: SetScrapeTargetConfig
-    ): RegionalSet {
-        throw NotImplementedException("Not implemented")
-    }
-
-    override fun extractMultiple(
-        element: Element,
-        providerConfig: ProviderConfig,
-        extractionConfig: SetScrapeTargetConfig
-    ): Collection<RegionalSet> {
-        val regionalSetList = extractAsListOfMaps(element, extractionConfig)
-        val cardPrintsInRegionalSet = extractAsListOfMaps(element, extractionConfig)
-
-        val cardPrints = cardPrintExtractionService.extractMultiple(
-            element,
-            providerConfig,
-            providerConfig.getTargets().getCardPrintConfiguration().get()
-        )
-
-        return regionalSetList.map {
-            RegionalSet.fromMap(it, cardPrints)
-        }
-    }
-}

src/main/kotlin/com/rak/service/ScrapeService.kt

@@ -1,20 +1,30 @@
 package com.rak.service
 
+import com.rak.config.model.ProviderConfig
 import com.rak.model.card.Card
+import com.rak.model.exception.NotImplementedException
+import com.rak.model.exception.TargetNotFoundException
 import com.rak.model.set.CardSet
 import com.rak.model.set.RegionalSet
+import com.rak.service.extract.RegionalSetExtractionService
+import com.rak.service.extract.CardSetExtractionService
+import io.quarkus.logging.Log
 import jakarta.enterprise.context.ApplicationScoped
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
+import java.lang.Exception
 
 @ApplicationScoped
 class ScrapeService(
     private val sourceService: SourceService,
-    private val extractionService: ExtractionService,
-    private val setExtractionService: SetExtractionService,
-    private val regionalSetExtractionService: RegionalSetExtractionService
+    private val cardSetExtractionService: CardSetExtractionService,
+    private val regionalSetExtractionService: RegionalSetExtractionService,
+    private val commonCrawlService: CommonCrawlService
 ) {
 
+    fun ProviderConfig.buildUrl(targetName: String): String {
+        return this.getUrlPattern().format(targetName)
+    }
 
     fun scrapeSet(
         provider: String,
@@ -23,10 +33,37 @@ class ScrapeService(
         val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found")
 
         val path: String = normalizePath(setName)
-        val document: Document = Jsoup.connect("https://${source.getDomain()}/$path").get()
+        val url = source.buildUrl(path)
+        val ccIndexResponses = commonCrawlService.queryAllCrawlIndices(url).sortedBy { it.timestamp }
 
-//        return extractionService.extractSet(setName, document, provider)
-        return setExtractionService.extract(document, source, source.getTargets().getSetConfig().get())
+        var document: Document? = null
+
+        for (indexResponse in ccIndexResponses) {
+            document = commonCrawlService.getDocument(
+                indexResponse,
+                source.getDomain()
+            )
+
+            if (document != null) {
+                break
+            }
+        }
+
+        if (document == null) {
+            // Fallback to Jsoup directly
+            try {
+                document = Jsoup.connect(url).get()
+            } catch(ex: Exception) {
+                Log.warn("Error occurred during Jsoup query", ex)
+                throw TargetNotFoundException("Could not find '$setName' for Provider '$provider'")
+            }
+        }
+
+        return cardSetExtractionService.extract(
+            document,
+            source,
+            source.getTargets().getSetConfig().get()
+        )
     }
 
     fun scrapeRegionalSet(
@@ -38,19 +75,15 @@ class ScrapeService(
         val path: String = normalizePath(setName)
         val document: Document = Jsoup.connect("https://${source.getDomain()}/$path").get()
 
-        return regionalSetExtractionService.extract(document, source, source.getTargets().getSetConfig().get())
+        return regionalSetExtractionService.extract(document, source, source.getTargets().getRegionalSetConfig().get())
     }
 
     fun scrapeCard(
         provider: String,
         cardName: String,
     ): Card? {
-        val source = sourceService.getSourceById(provider) ?: throw IllegalArgumentException("Provider $provider not found")
+        throw NotImplementedException("Not implemented")
 
-        val path: String = normalizePath(cardName)
-        val document: Document = Jsoup.connect("https://${source.getDomain()}/$path").get()
-
-        return extractionService.extractCard(document, provider)
     }
 
     private fun normalizePath(path: String): String = path

src/main/kotlin/com/rak/service/SourceService.kt

@@ -1,7 +1,7 @@
 package com.rak.service
 
 import com.rak.config.model.CardScrapeTargetConfig
-import com.rak.config.model.SetScrapeTargetConfig
+import com.rak.config.model.RegionalSetScrapeTargetConfig
 import com.rak.config.model.ProviderConfig
 import com.rak.config.model.SourcesConfig
 import com.rak.model.exception.InvalidConfigurationException
@@ -21,7 +21,7 @@ class SourceService(
     }
 
     private fun validateSource(providerConfig: ProviderConfig) {
-        val optionalRegionalSetConfig = providerConfig.getTargets().getSetConfig()
+        val optionalRegionalSetConfig = providerConfig.getTargets().getRegionalSetConfig()
         val optionalCardConfig = providerConfig.getTargets().getCardConfig()
 
         if (optionalRegionalSetConfig.isPresent) {
@@ -33,7 +33,7 @@ class SourceService(
         }
     }
 
-    private fun validateSetExtractConfig(setExtractConfig: SetScrapeTargetConfig) {
+    private fun validateSetExtractConfig(setExtractConfig: RegionalSetScrapeTargetConfig) {
         val selectors = listOf(
             setExtractConfig.getLanguageConfig(),
             setExtractConfig.getIdConfig(),

src/main/kotlin/com/rak/service/client/CommonCrawlRestClient.kt

@@ -0,0 +1,57 @@
+package com.rak.service.client
+
+import com.rak.util.NDJsonReader
+import com.rak.model.cc.CCIndexSuccessResponse
+import io.netty.buffer.ByteBufInputStream
+import io.quarkus.rest.client.reactive.ClientQueryParam
+import io.quarkus.rest.client.reactive.NotBody
+import io.quarkus.rest.client.reactive.Url
+import io.smallrye.faulttolerance.api.RateLimit
+import jakarta.ws.rs.Consumes
+import jakarta.ws.rs.GET
+import jakarta.ws.rs.Path
+import jakarta.ws.rs.PathParam
+import jakarta.ws.rs.QueryParam
+import org.eclipse.microprofile.faulttolerance.Bulkhead
+import org.eclipse.microprofile.rest.client.annotation.ClientHeaderParam
+import org.eclipse.microprofile.rest.client.annotation.RegisterProvider
+import org.eclipse.microprofile.rest.client.inject.RegisterRestClient
+import java.time.temporal.ChronoUnit
+
+@RegisterRestClient(baseUri = "whatever")
+@RegisterProvider(NDJsonReader::class)
+interface CommonCrawlRestClient {
+
+    @GET
+    @ClientQueryParam(name = "output", value = ["json"])
+    @Path("/{index}-index")
+    @Consumes("text/x-ndjson")
+    @RateLimit(
+        value = 1,
+        minSpacing = 5
+    )
+    @Bulkhead
+    fun queryIndex(
+        @Url
+        baseUrl: String,
+        @QueryParam("url")
+        queryUrl: String,
+        @PathParam("index")
+        indexName: String
+    ): CCIndexSuccessResponse
+
+    @GET
+    @Path("/{fileName}")
+    @ClientHeaderParam(name = "Range", value = ["{com.rak.util.HttpUtil.computeHeader}"])
+    fun getWarcArchive(
+        @Url
+        baseUrl: String,
+        @PathParam("fileName")
+        fileName: String,
+        @NotBody
+        fileLength: Int,
+        @NotBody
+        fileOffset: Int
+    ): ByteBufInputStream
+
+}

src/main/kotlin/com/rak/service/extract/AbstractExtractionService.kt

@@ -1,19 +1,17 @@
-package com.rak.service
+package com.rak.service.extract
 
-import com.rak.config.model.AbstractScrapeTargetConfig
-import com.rak.config.model.ExtractConfig
-import com.rak.config.model.ProviderConfig
-import com.rak.config.model.ScrapeTargetFieldConfig
-import com.rak.model.DiscriminatorDirection
+import com.rak.config.model.*
 import com.rak.model.Selector
 import com.rak.model.exception.ElementNotFoundException
 import com.rak.model.exception.InvalidConfigurationException
+import com.rak.model.exception.ValueValidationException
 import com.rak.model.transform.TransformationRegistry
 import com.rak.util.CssUtil
 import com.rak.util.XPathUtil
+import io.quarkus.logging.Log
 import org.jsoup.nodes.Element
 import org.jsoup.select.Elements
-import java.util.Optional
+import java.util.*
 import kotlin.jvm.optionals.getOrElse
 
 // find root element from global or node config
@@ -35,7 +33,13 @@ abstract class AbstractExtractionService<E, T : AbstractScrapeTargetConfig> {
         element: Element,
         providerConfig: ProviderConfig,
         extractionConfig: T
-    ): Collection<E>
+    ): List<E>
+
+    abstract fun extractNestedMultiples(
+        element: Element,
+        providerConfig: ProviderConfig,
+        extractionConfig: T
+    ): List<List<E>>
 
     fun getRootElement(
         element: Element,
@@ -84,7 +88,7 @@ abstract class AbstractExtractionService<E, T : AbstractScrapeTargetConfig> {
         }
     }
 
-    protected fun extractAsMap(
+    protected fun extractSingle(
         document: Element,
         extractionConfig: T
     ): Map<String, String> {
@@ -108,7 +112,7 @@ abstract class AbstractExtractionService<E, T : AbstractScrapeTargetConfig> {
         return result
     }
 
-    fun extractAsListOfMaps(
+    fun extractMulti(
         element: Element,
         extractionConfig: T
     ): List<Map<String, String>> {
@@ -126,7 +130,11 @@ abstract class AbstractExtractionService<E, T : AbstractScrapeTargetConfig> {
                 val extractedText = extractTextFromElementByTargetFieldConfig(
                     rootElement,
                     fieldConfig
-                ) ?: throw ElementNotFoundException("Could not find element for '$identifier'")
+                ) ?: if (fieldConfig.isNullable())  {
+                    ""
+                } else {
+                    throw ElementNotFoundException("Could not find element for '$identifier'")
+                }
 
                 val mapToModify: MutableMap<String, String> = try {
                     resultList[index]
@@ -143,95 +151,25 @@ abstract class AbstractExtractionService<E, T : AbstractScrapeTargetConfig> {
         return resultList
     }
 
-    fun extractAsListOfMaps(
-        elements: Elements,
-        extractionConfig: T
-    ): List<Map<String, String>> {
-        val resultList = mutableListOf<MutableMap<String, String>>()
-
-
-        // refactor this
-        extractionConfig.getItems().forEach { (identifier, fieldConfig) ->
-            for(index in 0..elements.size - 1) {
-                val rootElement = elements[index]
-                val extractedText = extractTextFromElementByTargetFieldConfig(
-                    rootElement,
-                    fieldConfig
-                ) ?: throw ElementNotFoundException("Could not find element for '$identifier'")
-
-                val mapToModify: MutableMap<String, String> = try {
-                    resultList[index]
-                } catch (_: IndexOutOfBoundsException) {
-                    val newMap = mutableMapOf<String, String>()
-                    resultList.add(newMap)
-                    newMap
-                }
-
-                mapToModify.put(identifier, extractedText)
-            }
-        }
-
-        return resultList
-    }
-
-    fun extractWithDiscriminator(
+    fun extractMultiWithDiscriminator(
         element: Element,
         extractionConfig: T
     ): List<List<Map<String, String>>>{
-        val rootElement = getRootElement(
+        val rootElements = getRootElements(
             element,
-            extractionConfig.getRootConfig(),
+            extractionConfig.getDiscriminator().get().getRootConfig(),
             Optional.empty<ExtractConfig>()
         )
 
-        var rootElements = getRootElements(
-            element,
-            extractionConfig.getRootConfig(),
-            Optional.empty<ExtractConfig>()
-        )
-
-        val discriminatedElements = getElementsFromElementByExtractConfig(
-            rootElement,
-            extractionConfig.getDiscriminator().get().getRootConfig().get(),
-        )
-
-        val discriminations = mutableListOf<String>()
         val result = mutableListOf<List<Map<String, String>>>()
 
-        for (element in discriminatedElements) {
-            val discriminatorValue: String = extractTextFromElementByTargetFieldConfig(
+        for(element in rootElements) {
+            result.add(extractMulti(
                 element,
-                extractionConfig.getDiscriminator().get()
-            ) ?: throw ElementNotFoundException("")
-
-            discriminations.add(discriminatorValue)
+                extractionConfig
+            ))
         }
 
-        val definitiveElements = if (discriminations.size < rootElements.size) {
-            if (extractionConfig.getDiscriminator().get().getDiscriminatorDirection() == DiscriminatorDirection.DESC) {
-                rootElements = Elements(rootElements.reversed())
-            }
-
-            while (discriminations.size < rootElements.size) {
-                rootElements.removeFirst()
-            }
-
-            if (extractionConfig.getDiscriminator().get().getDiscriminatorDirection() == DiscriminatorDirection.DESC) {
-                rootElements = Elements(rootElements.reversed())
-            }
-
-            rootElements
-        } else {
-            rootElements
-        }
-
-        result.add(extractAsListOfMaps(
-            definitiveElements,
-            extractionConfig
-        ))
-
-
-
         return result
     }
 
@@ -239,41 +177,87 @@ abstract class AbstractExtractionService<E, T : AbstractScrapeTargetConfig> {
         root: Element,
         extractionConfig: ScrapeTargetFieldConfig
     ): String? {
-        val extractionSteps = extractionConfig.getExtractionSteps()
-        val transformationSteps = extractionConfig.getOptionalTransformationSteps()
-
-        var currentElement: Element? = root.clone()
+        val extractionMethods = extractionConfig.getExtractionMethods()
         var result: String? = null
 
-        for (index in 0 until extractionSteps.size) {
-            val currentStep = extractionSteps.elementAtOrNull(index) ?: return null
-            if (currentElement == null) {
-                throw IllegalStateException()
-            }
 
-            if (index == extractionSteps.size - 1) {
-                result =  when (currentStep.selectorType()) {
-                    Selector.CSS -> CssUtil.extractResult(currentElement, currentStep.getQueryString())
-                    Selector.XPATH -> XPathUtil.extractResult(currentElement, currentStep.getQueryString())
+        for(extractionMethod in extractionMethods) {
+            val extractionSteps = extractionMethod.getExtractionSteps()
+            val transformationSteps = extractionMethod.getOptionalTransformationSteps()
+
+            var currentElement: Element? = root.clone()
+            var intermediateResult: String? = null
+
+            try {
+                for (index in 0 until extractionSteps.size) {
+                    val currentStep = extractionSteps.elementAtOrNull(index) ?: return null
+                    if (currentElement == null) {
+                        throw IllegalStateException()
+                    }
+
+                    if (index == extractionSteps.size - 1) {
+                        intermediateResult =  when (currentStep.selectorType()) {
+                            Selector.CSS -> CssUtil.extractResult(currentElement, currentStep.getQueryString())
+                            Selector.XPATH -> XPathUtil.extractResult(currentElement, currentStep.getQueryString())
+                        }
+                    }
+                    else {
+                        currentElement = when (currentStep.selectorType()) {
+                            Selector.CSS -> CssUtil.getNextElement(currentElement, currentStep.getQueryString())
+                            Selector.XPATH -> XPathUtil.getNextElement(currentElement, currentStep.getQueryString())
+                        }
+                    }
                 }
-            }
-            else {
-                currentElement = when (currentStep.selectorType()) {
-                    Selector.CSS -> CssUtil.getNextElement(currentElement, currentStep.getQueryString())
-                    Selector.XPATH -> XPathUtil.getNextElement(currentElement, currentStep.getQueryString())
+
+                if (intermediateResult == null) {
+                    throw ElementNotFoundException("Result could not be extracted")
+                } else {
+                    try {
+                        validateValue(intermediateResult, extractionConfig.getOptionalValidation())
+                    } catch (ex: ValueValidationException) {
+                        throw ex
+                    }
+
+                    if (transformationSteps.isPresent) {
+                        intermediateResult = transformationRegistry.applyTransformations(intermediateResult, transformationSteps.get())
+                    }
+
+                    result = intermediateResult
+                    break
+                }
+            } catch (ex: RuntimeException) {
+                when (ex) {
+                    is ElementNotFoundException,
+                    is IllegalStateException,
+                    is ValueValidationException -> Log.debug(ex.message)
+                    else -> throw ex
                 }
             }
         }
 
-        if (result == null) {
-            throw ElementNotFoundException("Result could not be extracted")
-        }
-
-        if (transformationSteps.isPresent) {
-            result = transformationRegistry.applyTransformations(result, transformationSteps.get())
+        if (result == null && extractionConfig.getFallbackConfiguration().isPresent) {
+            result = extractionConfig.getFallbackConfiguration().get().getOptionalDefaultValue()
         }
 
         return result
     }
 
+    private fun validateValue(value: String, validationConfig: Optional<ValidationConfig>) {
+        if (!validationConfig.isPresent) {
+            return
+        }
+
+        var validated = true
+
+        for(regex in validationConfig.get().getRegexPatterns()) {
+            if (!value.matches(regex.toRegex())) {
+                validated = false
+            }
+        }
+
+        if (!validated) {
+            throw ValueValidationException("'$value' does not validate against RegEx(s)")
+        }
+    }
+
 }

src/main/kotlin/com/rak/service/extract/CardPrintExtractionService.kt

@@ -1,15 +1,11 @@
-package com.rak.service
+package com.rak.service.extract
 
 import com.rak.config.model.CardPrintScrapeTargetConfig
 import com.rak.config.model.ProviderConfig
 import com.rak.config.model.ScrapeTargetFieldConfig
-import com.rak.config.model.SetScrapeTargetConfig
 import com.rak.model.card.CardPrint
 import com.rak.model.exception.NotImplementedException
-import com.rak.model.set.CardSet
-import com.rak.model.set.RegionalSet
 import jakarta.enterprise.context.ApplicationScoped
-import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 
 @ApplicationScoped
@@ -36,11 +32,22 @@ class CardPrintExtractionService : AbstractExtractionService<CardPrint, CardPrin
         element: Element,
         providerConfig: ProviderConfig,
         extractionConfig: CardPrintScrapeTargetConfig
-    ): Collection<CardPrint> {
-        val objectAsListOfMaps = extractWithDiscriminator(element, extractionConfig)
+    ): List<CardPrint> {
+        throw NotImplementedException("Not implemented")
+    }
 
-        return objectAsListOfMaps.map {
-            CardPrint.fromMap(it[0])
+    override fun extractNestedMultiples(
+        element: Element,
+        providerConfig: ProviderConfig,
+        extractionConfig: CardPrintScrapeTargetConfig
+    ): List<List<CardPrint>> {
+        val objectAsListOfMaps: List<List<Map<String, String>>> = extractMultiWithDiscriminator(
+            element,
+            extractionConfig
+        )
+
+        return objectAsListOfMaps.map { innerList ->
+            innerList.map { map -> CardPrint.fromMap(map) }
         }
     }
 }

src/main/kotlin/com/rak/service/extract/CardSetExtractionService.kt

@@ -1,4 +1,4 @@
-package com.rak.service
+package com.rak.service.extract
 
 import com.rak.config.model.ProviderConfig
 import com.rak.config.model.ScrapeTargetFieldConfig
@@ -9,15 +9,13 @@ import jakarta.enterprise.context.ApplicationScoped
 import org.jsoup.nodes.Element
 
 @ApplicationScoped
-class SetExtractionService(
+class CardSetExtractionService(
     private val regionalSetExtractionService: RegionalSetExtractionService
 ) : AbstractExtractionService<CardSet, SetScrapeTargetConfig>() {
 
     override fun SetScrapeTargetConfig.getItems(): Map<String, ScrapeTargetFieldConfig> {
         return mapOf(
-            Pair("prefix", this.getIdConfig()),
-            Pair("regionCode", this.getRegionKeyConfig()),
-            Pair("region", this.getLanguageConfig()),
+            Pair("name", this.getNameConfig()),
         )
     }
 
@@ -26,9 +24,15 @@ class SetExtractionService(
         providerConfig: ProviderConfig,
         extractionConfig: SetScrapeTargetConfig
     ): CardSet {
-        return CardSet(
-            "test",
-            regionalSetExtractionService.extractMultiple(element, providerConfig, extractionConfig).toSet()
+        val set = extractSingle(element, extractionConfig)
+
+        return CardSet.fromMap(
+            set,
+            regionalSetExtractionService.extractMultiple(
+                element,
+                providerConfig,
+                providerConfig.getTargets().getRegionalSetConfig().get()
+            ).toSet()
         )
     }
 
@@ -36,7 +40,15 @@ class SetExtractionService(
         element: Element,
         providerConfig: ProviderConfig,
         extractionConfig: SetScrapeTargetConfig
-    ): Collection<CardSet> {
+    ): List<CardSet> {
+        throw NotImplementedException("Not implemented")
+    }
+
+    override fun extractNestedMultiples(
+        element: Element,
+        providerConfig: ProviderConfig,
+        extractionConfig: SetScrapeTargetConfig
+    ): List<List<CardSet>> {
         throw NotImplementedException("Not implemented")
     }
 }

src/main/kotlin/com/rak/service/extract/RegionalSetExtractionService.kt

@@ -0,0 +1,62 @@
+package com.rak.service.extract
+
+import com.rak.config.model.ProviderConfig
+import com.rak.config.model.ScrapeTargetFieldConfig
+import com.rak.config.model.RegionalSetScrapeTargetConfig
+import com.rak.config.model.SourcesConfig
+import com.rak.model.card.CardPrint
+import com.rak.model.exception.NotImplementedException
+import com.rak.model.set.RegionalSet
+import jakarta.enterprise.context.ApplicationScoped
+import org.jsoup.nodes.Element
+
+@ApplicationScoped
+class RegionalSetExtractionService(
+    private val cardPrintExtractionService: CardPrintExtractionService,
+    private val sourcesConfig: SourcesConfig
+) : AbstractExtractionService<RegionalSet, RegionalSetScrapeTargetConfig>() {
+
+    override fun RegionalSetScrapeTargetConfig.getItems(): Map<String, ScrapeTargetFieldConfig> {
+        return mapOf(
+            Pair("prefix", this.getIdConfig()),
+            Pair("regionCode", this.getRegionKeyConfig()),
+            Pair("region", this.getLanguageConfig()),
+        )
+    }
+
+    override fun extract(
+        element: Element,
+        providerConfig: ProviderConfig,
+        extractionConfig: RegionalSetScrapeTargetConfig
+    ): RegionalSet {
+        throw NotImplementedException("Not implemented")
+    }
+
+    override fun extractMultiple(
+        element: Element,
+        providerConfig: ProviderConfig,
+        extractionConfig: RegionalSetScrapeTargetConfig
+    ): List<RegionalSet> {
+        val regionalSetList = extractMulti(element, extractionConfig)
+
+        val cardPrintGroups: List<List<CardPrint>> = cardPrintExtractionService.extractNestedMultiples(
+            element,
+            providerConfig,
+            providerConfig.getTargets().getCardPrintConfiguration().get()
+        )
+
+        // Pair each RegionalSet with its CardPrint group by index
+        return regionalSetList.mapIndexed { index, regionalSetMap ->
+            val cardPrintsForSet = cardPrintGroups.getOrElse(index) { emptyList() }
+            RegionalSet.fromMap(regionalSetMap, cardPrintsForSet)
+        }
+    }
+
+    override fun extractNestedMultiples(
+        element: Element,
+        providerConfig: ProviderConfig,
+        extractionConfig: RegionalSetScrapeTargetConfig
+    ): List<List<RegionalSet>> {
+        throw NotImplementedException("Not implemented")
+    }
+}

src/main/kotlin/com/rak/util/HttpUtil.kt

@@ -0,0 +1,23 @@
+package com.rak.util
+
+import io.quarkus.rest.client.reactive.ComputedParamContext
+
+class HttpUtil {
+
+    companion object {
+
+        private const val HEADER_FORMAT_STRING: String = "bytes=%d-%d"
+
+        @JvmStatic
+        fun computeHeader(context: ComputedParamContext): String {
+            val fileLengthContext = context.methodParameters().subList(2, 4)
+
+            val fileLength = fileLengthContext[0].value().toString().toInt()
+            val fileOffset = fileLengthContext[1].value().toString().toInt()
+
+            return HEADER_FORMAT_STRING.format(fileOffset, fileOffset + fileLength - 1)
+        }
+
+    }
+
+}

src/main/kotlin/com/rak/util/NDJsonReader.kt

@@ -0,0 +1,44 @@
+package com.rak.util
+
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.rak.model.cc.CCIndexSuccessResponse
+import jakarta.ws.rs.Consumes
+import jakarta.ws.rs.core.MediaType
+import jakarta.ws.rs.core.MultivaluedMap
+import jakarta.ws.rs.ext.MessageBodyReader
+import jakarta.ws.rs.ext.Provider
+import java.io.BufferedReader
+import java.io.InputStream
+import java.io.InputStreamReader
+import java.lang.reflect.Type
+
+@Provider
+@Consumes("text/x-ndjson") // Handles NDJSON content
+class NDJsonReader : MessageBodyReader<CCIndexSuccessResponse> {
+
+    private val objectMapper = jacksonObjectMapper().registerModule(JavaTimeModule())
+
+    override fun isReadable(
+        type: Class<*>?,
+        genericType: Type?,
+        annotations: Array<out Annotation>?,
+        mediaType: MediaType?
+    ): Boolean {
+        return type == CCIndexSuccessResponse::class.java
+    }
+
+    override fun readFrom(
+        type: Class<CCIndexSuccessResponse>,
+        genericType: Type?,
+        annotations: Array<out Annotation>?,
+        mediaType: MediaType?,
+        httpHeaders: MultivaluedMap<String, String>?,
+        entityStream: InputStream
+    ): CCIndexSuccessResponse {
+        BufferedReader(InputStreamReader(entityStream)).use { reader ->
+            val firstLine = reader.readLine()
+            return objectMapper.readValue(firstLine, CCIndexSuccessResponse::class.java)
+        }
+    }
+}

src/main/kotlin/com/rak/util/XPathUtil.kt

@@ -3,12 +3,16 @@ package com.rak.util
 import com.rak.model.XPathTarget
 import org.jsoup.nodes.Element
 import org.jsoup.nodes.TextNode
+import org.jsoup.select.Elements
+import java.util.regex.Pattern
+import kotlin.coroutines.CoroutineContext
 
 class XPathUtil private constructor() {
 
     companion object {
         private val TEXT_NODE_MATCHER: Regex = Regex("^.*text\\(\\)$")
         private val ATTRIBUTE_MATCHER: Regex = Regex("^//[/a-z]*@([a-z]*)$")
+        private val INDEX_MATCHER: Regex = Regex("\\[(\\w)\\]")
 
         private fun extractTextFromAttribute(root: Element, xpath: String): String? {
             val groupMatcher = ATTRIBUTE_MATCHER.matchEntire(xpath)
@@ -20,14 +24,29 @@ class XPathUtil private constructor() {
             }
         }
 
+        private fun selectXpath(element: Element, xpath: String): Elements {
+            return if (xpath.contains(INDEX_MATCHER)) {
+                val index = INDEX_MATCHER.find(xpath)?.groupValues[1]!!.toInt()
+                val xpathHalves = xpath.split("[$index]")
+
+                try {
+                    Elements(element.selectXpath(xpathHalves[0])[index])
+                } catch (_: IndexOutOfBoundsException) {
+                    Elements()
+                }
+            } else {
+                element.selectXpath(xpath)
+            }
+        }
+
         private fun extractTextFromNode(root: Element, xpath: String): String? {
             return root
-                .selectXpath(xpath, TextNode::class.java)
-                .firstOrNull()?.text()
+                .selectXpath(xpath.replace("/text()", ""))
+                .text()
         }
 
         fun getNextElement(element: Element, path: String): Element? {
-            return element.selectXpath(path).firstOrNull()
+            return selectXpath(element, path).firstOrNull()
         }
 
         fun extractResult(root: Element, path: String): String? {

src/main/resources/application.yml

@@ -1,121 +1,210 @@
 quarkus:
+
+  container-image:
+    registry: gitea.smoothbrain.win
+    group: rak
+    build: true
+    additional-tags: latest
+
   http:
     port: 8081
+  live-reload:
+    instrumentation: true
 
 scraper:
   sources:
-    - id: konami-official
-      name: "Konami Official Database"
-      domain: "yugioh-card.com"
-      url-patterns:
-        - "^https://www\\.yugioh-card\\.com/[a-z]{2}/products/.*$"
-      targets:
-        card:
-          root:
-            type: css
-            value: "h3:contains(Prefix(es)) + div > ul:nth-child(1) > li"
-          name:
-            steps:
-              - type: "css"
-                value: "h1.product-title"
-              - type: "xpath"
-                value: "//h1[@itemprop='name']"
-          attack:
-            steps:
-              - type: "css"
-                value: ".atk-value"
+#    - id: konami-official
+#      name: "Konami Official Database"
+#      domain: "yugioh-card.com"
+#      url-pattern: "^https://www\\.yugioh-card\\.com/[a-z]{2}/products/.*$"
+#      targets:
+#        card:
+#          root:
+#            type: css
+#            value: "h3:contains(Prefix(es)) + div > ul:nth-child(1) > li"
+#          name:
+#            steps:
+#              - type: "css"
+#                value: "h1.product-title"
+#              - type: "xpath"
+#                value: "//h1[@itemprop='name']"
+#          attack:
+#            steps:
+#              - type: "css"
+#                value: ".atk-value"
 
     - id: ygo-fandom
       name: "Yu-Gi-Oh Fandom Wiki"
       domain: "yugioh.fandom.com"
-      url-patterns:
-        - "^https://yugioh\\.fandom\\.com/wiki/.*$"
+      url-pattern: "https://yugioh.fandom.com/wiki/%s"
       targets:
         set:
+          root:
+            type: css
+            value: "aside > .pi-title"
+          name:
+            type: string
+            extractors:
+              - steps:
+                  - type: xpath
+                    value: "//h2/text()"
+        regional-set:
           root:
             type: css
             value: "h3:contains(Prefix(es)) + div > ul:nth-child(1) > li"
           id:
-            steps:
-              - type: xpath
-                value: "//li/text()"
-            transform:
-              - name: "replace"
-                parameters:
-                  - " ("
-                  - ""
+            type: int
+            extractors:
+              - steps:
+                - type: xpath
+                  value: "//li/text()"
+                transform:
+                  - name: "regexReplace"
+                    parameters: [
+                      " *\\(.+\\)",
+                      ""
+                    ]
           language:
-            steps:
-              - type: xpath
-                value: "//li/abbr"
-              - type: xpath
-                value: "//abbr/@title"
+            type: int
+            extractors:
+              - steps:
+                - type: xpath
+                  value: "//li/abbr"
+                - type: xpath
+                  value: "//abbr/@title"
           region-key:
-            steps:
-              - type: xpath
-                value: "//li/abbr/text()"
+            type: int
+            extractors:
+              - steps:
+                - type: xpath
+                  value: "//li/abbr/text()"
         card-print:
           multi: true
           root:
             type: css
-            value: ".tabber.wds-tabber > div"
+            value: "table > tbody > tr:has(> td)"
           discriminator:
-            direction: asc
+            type: string
             root:
               type: css
-              value: ".wds-tabs__tab"
-            steps:
-              - type: xpath
-                value: "//li/div/a/text()"
+              value: ".wds-tab__content"
           id:
-            steps:
-              - type: xpath
-                value: ".//table/tbody/tr[2]/td[1]/a/text()"
+            type: int
+            extractors:
+              - steps:
+                - type: xpath
+                  value: "./td/a[0]"
+                - type: xpath
+                  value: "./text()"
+                transform:
+                  - name: "regexReplace"
+                    parameters: [
+                      ".+-[A-Za-z]*0?",
+                      ""
+                    ]
+              - steps:
+                - type: xpath
+                  value: "./td/span/text()"
+                transform:
+                  - name: "regexReplace"
+                    parameters: [
+                      " .+",
+                      ""
+                    ]
+                  - name: "regexReplace"
+                    parameters: [
+                      ".+-[A-Za-z]*0?",
+                      ""
+                    ]
+            validation:
+              pattern: "^.+-.+\\\\d.+$"
           name:
-            steps:
-              - type: xpath
-                value: ".//table/tbody/tr[2]/td[1]/a/text()"
+            type: int
+            extractors:
+              - steps:
+                - type: xpath
+                  value: "./td[1]"
+                - type: xpath
+                  value: "./text()"
+                transform:
+                  - name: "regexReplace"
+                    parameters: [
+                      " ?\\(.+\\)",
+                      ""
+                    ]
+                  - name: "removeInnerQuotes"
+                    parameters: []
+            validation:
+              pattern: "^\".+\".*"
           regional-name:
-            steps:
-              - type: xpath
-                value: ".//table/tbody/tr[2]/td[2]/a/text()"
+            type: int
+            nullable: true
+            extractors:
+              - steps:
+                - type: xpath
+                  value: "./td[2]"
+                - type: xpath
+                  value: "./text()"
+                transform:
+                  - name: "removeInnerQuotes"
+                    parameters: []
+            validation:
+              pattern: "^\".+\"$"
           rarity:
-            steps:
-              - type: xpath
-                value: ".//table/tbody/tr[2]/td[3]/a/text()"
-        card:
-          name:
-            root:
-              type: css
-              value: ".cardTable"
-            steps:
-              - type: "xpath"
-                value: "./tbody/tr[3]/th/text()"
-          description:
-            root:
-              type: css
-              value: ".cardTable"
-            steps:
-              - type: "xpath"
-                value: "b:contains(Card descriptions)"
-          type:
-            root:
-              type: css
-              value: ".cardTable"
-            steps:
-              - type: "xpath"
-                value: "b:contains(Card descriptions)"
-          attack:
-            root:
-              type: css
-              value: ".cardTable"
-            steps:
-              - type: "xpath"
-                value: "b:contains(Card descriptions)"
-          defense:
-            root:
-              type: css
-              value: ".cardTable"
-            steps:
-              - type: "xpath"
-                value: "b:contains(Card descriptions)"
+            fallback:
+              default: "N/A"
+            type: int
+            extractors:
+              - steps:
+                - type: xpath
+                  value: "./td/a[3]"
+                - type: xpath
+                  value: "./text()"
+              - steps:
+                - type: xpath
+                  value: "./td/a[2]"
+                - type: xpath
+                  value: "./text()"
+              - steps:
+                - type: xpath
+                  value: "./td/a[1]"
+                - type: xpath
+                  value: "./text()"
+            validation:
+              pattern: "^.*(Common|Rare|Print).*$"
+#        card:
+#          name:
+#            root:
+#              type: css
+#              value: ".cardTable"
+#            steps:
+#              - type: "xpath"
+#                value: "./tbody/tr[3]/th/text()"
+#          description:
+#            root:
+#              type: css
+#              value: ".cardTable"
+#            steps:
+#              - type: "xpath"
+#                value: "b:contains(Card descriptions)"
+#          type:
+#            root:
+#              type: css
+#              value: ".cardTable"
+#            steps:
+#              - type: "xpath"
+#                value: "b:contains(Card descriptions)"
+#          attack:
+#            root:
+#              type: css
+#              value: ".cardTable"
+#            steps:
+#              - type: "xpath"
+#                value: "b:contains(Card descriptions)"
+#          defense:
+#            root:
+#              type: css
+#              value: ".cardTable"
+#            steps:
+#              - type: "xpath"
+#                value: "b:contains(Card descriptions)"

src/test/kotlin/com/rak/ExampleResourceTest.kt

@@ -1,20 +0,0 @@
-package com.rak
-
-import io.quarkus.test.junit.QuarkusTest
-import io.restassured.RestAssured.given
-import org.hamcrest.CoreMatchers.`is`
-import org.junit.jupiter.api.Test
-
-@QuarkusTest
-class ExampleResourceTest {
-
-    @Test
-    fun testHelloEndpoint() {
-        given()
-            .`when`().get("/hello")
-            .then()
-            .statusCode(200)
-            .body(`is`("Hello from Quarkus REST"))
-    }
-
-}