diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..cd36b69 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,32 @@ +name: Create and Push Release +on: + workflow_dispatch: + +env: + AUTHENTIK_URL: https://auth.smoothbrain.win + REGISTRY_URL: gitea.smoothbrain.win + IMAGE_OWNER: rak + IMAGE_NAME: dex-scraper-java + +jobs: + + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup JDK + uses: https://gitea.smoothbrain.win/rak/setup-java@main + with: + distribution: 'corretto' + java-version: '21.0.6' + cache: 'gradle' + + - name: Build & Push Image + env: + QUARKUS_CONTAINER_IMAGE_USERNAME: ${{ secrets.CI_SERVICE_ACCOUNT }} + QUARKUS_CONTAINER_IMAGE_PASSWORD: ${{ secrets.CI_SERVICE_ACCOUNT_PASSWORD }} + run: | + ./gradlew clean build \ + -Dquarkus.container-image.push=true \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 1210b3e..a05f2a0 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,6 +22,7 @@ dependencies { implementation("io.quarkus:quarkus-rest-client-kotlin-serialization") implementation("io.quarkus:quarkus-rest-jackson") implementation("io.quarkus:quarkus-kotlin") + implementation("io.quarkus:quarkus-smallrye-fault-tolerance") implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") implementation("io.quarkus:quarkus-arc") implementation("org.jsoup:jsoup:1.20.1") diff --git a/src/main/kotlin/com/rak/service/CommonCrawlService.kt b/src/main/kotlin/com/rak/service/CommonCrawlService.kt index 11c5a41..d50e25c 100644 --- a/src/main/kotlin/com/rak/service/CommonCrawlService.kt +++ b/src/main/kotlin/com/rak/service/CommonCrawlService.kt @@ -46,7 +46,7 @@ class CommonCrawlService( crawlName.indexName )) } catch (ex: RuntimeException) { - Log.warn("Error occurred querying crawl '${crawlName.indexName}' for URL $url") + Log.warn("Error occurred querying crawl '${crawlName.indexName}' for URL $url", ex) } } diff --git a/src/main/kotlin/com/rak/service/ScrapeService.kt b/src/main/kotlin/com/rak/service/ScrapeService.kt index c368a06..1522104 100644 --- a/src/main/kotlin/com/rak/service/ScrapeService.kt +++ b/src/main/kotlin/com/rak/service/ScrapeService.kt @@ -54,7 +54,7 @@ class ScrapeService( try { document = Jsoup.connect(url).get() } catch(ex: Exception) { - Log.warn("Error occurred during Jsoup query") + Log.warn("Error occurred during Jsoup query", ex) throw TargetNotFoundException("Could not find '$setName' for Provider '$provider'") } } diff --git a/src/main/kotlin/com/rak/service/client/CommonCrawlRestClient.kt b/src/main/kotlin/com/rak/service/client/CommonCrawlRestClient.kt index a8332e5..81399af 100644 --- a/src/main/kotlin/com/rak/service/client/CommonCrawlRestClient.kt +++ b/src/main/kotlin/com/rak/service/client/CommonCrawlRestClient.kt @@ -6,14 +6,17 @@ import io.netty.buffer.ByteBufInputStream import io.quarkus.rest.client.reactive.ClientQueryParam import io.quarkus.rest.client.reactive.NotBody import io.quarkus.rest.client.reactive.Url +import io.smallrye.faulttolerance.api.RateLimit import jakarta.ws.rs.Consumes import jakarta.ws.rs.GET import jakarta.ws.rs.Path import jakarta.ws.rs.PathParam import jakarta.ws.rs.QueryParam +import org.eclipse.microprofile.faulttolerance.Bulkhead import org.eclipse.microprofile.rest.client.annotation.ClientHeaderParam import org.eclipse.microprofile.rest.client.annotation.RegisterProvider import org.eclipse.microprofile.rest.client.inject.RegisterRestClient +import java.time.temporal.ChronoUnit @RegisterRestClient(baseUri = "whatever") @RegisterProvider(NDJsonReader::class) @@ -23,6 +26,11 @@ interface CommonCrawlRestClient { @ClientQueryParam(name = "output", value = ["json"]) @Path("/{index}-index") @Consumes("text/x-ndjson") + @RateLimit( + value = 1, + minSpacing = 5 + ) + @Bulkhead fun queryIndex( @Url baseUrl: String, diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 232155b..af23bfe 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,4 +1,11 @@ quarkus: + + container-image: + registry: gitea.smoothbrain.win + group: rak + build: true + additional-tags: latest + http: port: 8081 live-reload: