diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 9e7f59f..2d4696c 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -11,14 +11,16 @@ plugins { object Versions { const val jvmTarget = "11" const val kotlinTarget = "1.6" + const val kotlin = "1.6.10" const val gradleNodePlugin = "3.2.0" const val gradleDockerComposePlugin = "0.15.0" - const val kotlin = "1.6.10" const val kotlinXSerialization = "1.3.2" const val kvision = "5.8.1" const val semver = "1.1.2" + const val http4k = "4.19.1.0" + const val jsoup = "1.14.3" const val kotest = "5.1.0" } @@ -31,7 +33,7 @@ dependencies { implementation("org.jetbrains.kotlin:kotlin-reflect") implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:${Versions.kotlin}") - implementation(enforcedPlatform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}")) + implementation(platform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}")) implementation("org.jetbrains.kotlinx:kotlinx-serialization-core") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json") // implementation("com.charleskorn.kaml:kaml:0.37.0") @@ -46,6 +48,12 @@ dependencies { // https://github.com/avast/gradle-docker-compose-plugin implementation("com.avast.gradle:gradle-docker-compose-plugin:${Versions.gradleDockerComposePlugin}") + + implementation(platform("org.http4k:http4k-bom:${Versions.http4k}")) + implementation("org.http4k:http4k-core") + implementation("org.http4k:http4k-client-okhttp") + + implementation("org.jsoup:jsoup:${Versions.jsoup}") } diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts index 8065bb4..f5a0553 100644 --- a/buildSrc/settings.gradle.kts +++ b/buildSrc/settings.gradle.kts @@ -1,3 +1,13 @@ + rootProject.name = "buildSrc" apply(from = "./repositories.settings.gradle.kts") + +@Suppress("UnstableApiUsage") // Central declaration of repositories is an incubating feature +dependencyResolutionManagement { + versionCatalogs { + create("libs") { + from(files("../gradle/libs.versions.toml")) + } + } +} diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/FactorioModPortalDeploy.kt b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/FactorioModPortalDeploy.kt new file mode 100644 index 0000000..73ca284 --- /dev/null +++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/FactorioModPortalDeploy.kt @@ -0,0 +1,23 @@ +package dev.adamko.kafkatorio.task + +import org.gradle.api.DefaultTask +import org.http4k.client.OkHttp +import org.http4k.core.Method +import org.http4k.core.Request + +abstract class FactorioModPortalDeploy : DefaultTask() { + + init { + super.doFirst { + + val client = OkHttp() + + val request = Request(Method.GET, "https://factorio.com/login?mods=1") + + println(client(request)) + + + } + } + +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a2ff04b..bd0d438 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -90,10 +90,12 @@ gcp-speech = { group = "com.google.cloud", name = "google-cloud-speech" ## kafka ## kafka-streams = { group = "org.apache.kafka", name = "kafka-streams", version.ref = "kafka" } - # kafka helpers kotka = { group = "com.github.blueanvil", name = "kotka", version = "1.1.1" } +jsoup = { group = "org.jsoup", name = "jsoup", version = "1.14.3" } +skrapeit = { group = "it.skrape", name = "skrapeit", version = "1.2.0" } + ############# ### test ### diff --git a/modules/events-server/build.gradle.kts b/modules/events-server/build.gradle.kts index 8e140b2..be8c3c1 100644 --- a/modules/events-server/build.gradle.kts +++ b/modules/events-server/build.gradle.kts @@ -40,14 +40,9 @@ dependencies { implementation("com.sksamuel.scrimage:scrimage-core:4.0.24") val kotkaVersion = "v0.0.12" -// val kotkaVersion = "store-iterators-SNAPSHOT" implementation("com.github.adamko-dev.kotka-streams:kotka-streams:$kotkaVersion") { isChanging = true } -// val kotkaVersion = "main-SNAPSHOT" -// implementation("dev.adamko.kotka:kotka-streams:$kotkaVersion") { -// isChanging = true -// } } diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt index c1784f6..3f578e1 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt @@ -10,13 +10,14 @@ import org.apache.kafka.clients.admin.CreateTopicsResult import org.apache.kafka.clients.admin.NewTopic import org.apache.kafka.clients.admin.TopicListing -private val logger = KotlinLogging.logger {} //fun main() { // KafkaAdmin.createKafkatorioTopics() //} -object KafkaAdmin { +object KafkatorioKafkaAdmin { + + private val logger = KotlinLogging.logger {} private val kafkaAdmin: Admin = Admin.create( diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt index efba5aa..26269b1 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt @@ -5,6 +5,7 @@ import dev.adamko.kafkatorio.events.schema.FactorioEvent import dev.adamko.kafkatorio.events.schema.FactorioObjectData import dev.adamko.kafkatorio.events.schema.FactorioPrototypes import dev.adamko.kafkatorio.events.schema.KafkatorioPacket +import dev.adamko.kafkatorio.processor.config.ApplicationProperties import dev.adamko.kafkatorio.processor.serdes.jsonMapper import dev.adamko.kafkatorio.processor.serdes.kxsBinary import dev.adamko.kafkatorio.processor.topology.FactorioServerId @@ -36,8 +37,9 @@ import org.apache.kafka.streams.kstream.KStream import org.apache.kafka.streams.kstream.KTable -class KafkatorioTopology( +internal class KafkatorioTopology( private val websocketServer: WebsocketServer, + private val appProps: ApplicationProperties = ApplicationProperties(), ) : CoroutineScope { override val coroutineContext: CoroutineContext = diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt index a7f523e..e9ec6e3 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt @@ -1,8 +1,10 @@ package dev.adamko.kafkatorio.processor.config import dev.adamko.kafkatorio.processor.config.PortVal.Companion.portVal +import java.nio.file.Path import java.time.Clock import java.util.Properties +import kotlin.io.path.Path import org.http4k.cloudnative.env.Environment import org.http4k.cloudnative.env.EnvironmentKey import org.http4k.lens.nonEmptyString @@ -15,19 +17,21 @@ import org.http4k.lens.nonEmptyString * # 3. A resources-dir yaml file * */ -class ApplicationProperties( +internal class ApplicationProperties( vararg environments: Environment = arrayOf( Environment.env(), Environment.fromEnvVarPath(), Environment.fromResourcesFile(".secret.config.yml") ) ) { + + private val appProps: Environment = environments.reduce { acc, environment -> acc.overrides(environment) } + fun clock(): Clock = Clock.systemUTC() - // /** * The port of the server @@ -37,10 +41,6 @@ class ApplicationProperties( val serverPort: PortVal get() = EnvironmentKey.portVal().defaulted("PORT", PortVal.DEFAULT)[appProps] - // - - - // val kafkaConfig: Properties get() = EnvironmentKey @@ -54,6 +54,11 @@ class ApplicationProperties( } .required("kafka.config")[appProps] - // + + val webmapTileDir: Path + get() = EnvironmentKey + .nonEmptyString() + .map(::Path) + .required("tileServer.webmapTileDir")[appProps] } diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt index 58d592e..4bfeec7 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt @@ -13,11 +13,11 @@ import org.http4k.lens.Lens * * (An improved version of [Environment.fromResource]) */ -internal fun Environment.Companion.fromResourcesFile( +fun Environment.Companion.fromResourcesFile( resource: String, default: Environment = EMPTY -) = - try { +): Environment { + return try { val file = this::class.java .getResource("/${resource.removePrefix("/")}") @@ -27,6 +27,7 @@ internal fun Environment.Companion.fromResourcesFile( } catch (e: FileNotFoundException) { default } +} /** * Try fetching a properties file the location provided by the @@ -35,7 +36,7 @@ internal fun Environment.Companion.fromResourcesFile( * * If it does not exist, return [default]. */ -internal fun Environment.Companion.fromEnvVarPath( +fun Environment.Companion.fromEnvVarPath( location: Lens = Lenses.APPLICATION_PROPERTIES_FILE_PATH, default: Environment = EMPTY, ): Environment { @@ -65,7 +66,7 @@ internal fun Environment.Companion.fromFile( } /** The same as [Environment.ENV], but refreshes on each invocation (which is useful for testing). */ -internal fun Environment.Companion.env(): Environment = +fun Environment.Companion.env(): Environment = MapEnvironment.from(System.getenv().toProperties()) /** The same as [Environment.JVM_PROPERTIES], but refreshes on each invocation (which is useful for testing). */ diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt index 8e0c334..e53bf6d 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt @@ -1,7 +1,7 @@ package dev.adamko.kafkatorio.processor -import dev.adamko.kafkatorio.admin.KafkaAdmin +import dev.adamko.kafkatorio.admin.KafkatorioKafkaAdmin import dev.adamko.kafkatorio.processor.config.ApplicationProperties import dev.adamko.kafkatorio.processor.tileserver.WebMapTileServer import kotlinx.coroutines.runBlocking @@ -10,12 +10,12 @@ import org.http4k.server.Undertow import org.http4k.server.asServer -val appProps = ApplicationProperties() +//private val appProps = ApplicationProperties() fun main(): Unit = runBlocking { - KafkaAdmin.createKafkatorioTopics() + KafkatorioKafkaAdmin.createKafkatorioTopics() val wsServer = WebsocketServer() val tileServer = WebMapTileServer() diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt index db5e420..db9f440 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt @@ -1,5 +1,7 @@ package dev.adamko.kafkatorio.processor.tileserver +import dev.adamko.kafkatorio.processor.config.ApplicationProperties +import kotlin.io.path.absolutePathString import org.http4k.core.HttpHandler import org.http4k.core.then import org.http4k.filter.CachingFilters @@ -9,17 +11,13 @@ import org.http4k.routing.bind import org.http4k.routing.routes import org.http4k.routing.static -class WebMapTileServer { +internal class WebMapTileServer( + appProps: ApplicationProperties = ApplicationProperties() +) { private val routes: RoutingHttpHandler = routes( "/tiles" bind CachingFilters.Response.NoCache() - .then( - static( - ResourceLoader.Directory( - """D:\Users\Adam\Projects\games\kafkatorio\modules\events-server\src\main\resources\kafkatorio-web-map\""" - ) - ) - ) + .then(static(ResourceLoader.Directory(appProps.webmapTileDir.absolutePathString()))) ) fun build(): HttpHandler { diff --git a/modules/factorio-mod-portal-client/build.gradle.kts b/modules/factorio-mod-portal-client/build.gradle.kts new file mode 100644 index 0000000..0f3cb8d --- /dev/null +++ b/modules/factorio-mod-portal-client/build.gradle.kts @@ -0,0 +1,37 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + dev.adamko.kafkatorio.lang.`kotlin-jvm` + kotlin("plugin.serialization") + application +} + + +dependencies { + + implementation(projects.modules.eventsServer) + + implementation(platform(libs.kotlinx.serialization.bom)) + implementation(libs.kotlinx.serialization.core) + implementation(libs.kotlinx.serialization.json) + + implementation(libs.kotlinx.coroutines) + + implementation(platform(libs.http4k.bom)) + implementation(libs.bundles.http4k) + implementation(libs.http4k.format.kotlinx) + + implementation(libs.skrapeit) +} + +// +//application { +// mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt") +//} + +tasks.withType { + kotlinOptions.freeCompilerArgs += listOf( + "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi", + "-opt-in=kotlinx.coroutines.FlowPreview", + ) +} diff --git a/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt new file mode 100644 index 0000000..be38903 --- /dev/null +++ b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt @@ -0,0 +1,48 @@ +package dev.adamko.kafkatorio + +import dev.adamko.kafkatorio.processor.config.env +import dev.adamko.kafkatorio.processor.config.fromEnvVarPath +import dev.adamko.kafkatorio.processor.config.fromResourcesFile +import java.time.Clock +import org.http4k.cloudnative.env.Environment +import org.http4k.cloudnative.env.EnvironmentKey +import org.http4k.cloudnative.env.Secret +import org.http4k.lens.nonEmptyString +import org.http4k.lens.secret + + +/** + * Load properties, in descending priority order, from + * + * 1. Environment variables + * 2. This app's env file (as defined by the environment variable [Lenses.APPLICATION_PROPERTIES_FILE_PATH]) + * # 3. A resources-dir yaml file + * + */ +internal class ApplicationProperties( + vararg environments: Environment = arrayOf( + Environment.env(), + Environment.fromEnvVarPath(), + Environment.fromResourcesFile(".secret.config.yml") + ) +) { + + private val appProps: Environment = + environments.reduce { acc, environment -> acc.overrides(environment) } + + + fun clock(): Clock = Clock.systemUTC() + + + val factorioModPortalUsername: String + get() = EnvironmentKey + .nonEmptyString() + .required("factorio-mod-portal.username")[appProps] + + + val factorioModPortalPassword: Secret + get() = EnvironmentKey + .secret() + .required("factorio-mod-portal.password")[appProps] + +} diff --git a/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt new file mode 100644 index 0000000..55fb5e2 --- /dev/null +++ b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt @@ -0,0 +1,107 @@ +package dev.adamko.kafkatorio + +import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString +import dev.adamko.kafkatorio.processor.serdes.jsonMapper +import it.skrape.core.htmlDocument +import it.skrape.selects.html5.input +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import org.http4k.client.OkHttp +import org.http4k.core.HttpHandler +import org.http4k.core.Method +import org.http4k.core.Request +import org.http4k.core.Response +import org.http4k.core.body.form +import org.http4k.core.cookie.Cookie +import org.http4k.core.cookie.cookie +import org.http4k.core.cookie.cookies +import org.http4k.core.then +import org.http4k.filter.ClientFilters + + +/** + * * https://github.com/shanemadden/factorio-mod-portal-publish/blob/master/entrypoint.sh + * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/entrypoint.py + * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/factorioModPortal.py + * * https://github.com/justarandomgeek/vscode-factoriomod-debug/blob/master/src/ModPackageProvider.ts#L530 + * + * + */ +internal class FactorioModPortalClient( + private val appProps: ApplicationProperties = ApplicationProperties(), + private val loginFormUrl: String = "https://factorio.com/login?mods=1", + private val loginUrl: String = "https://factorio.com/login", +) { + + private val client: HttpHandler = + ClientFilters.FollowRedirects() + .then(OkHttp()) + + private val csrfCookie: Cookie by lazy { fetchCsrfToken() } + private val sessionCookie: Cookie by lazy { login() } + + + private fun fetchCsrfToken(): Cookie { + + val request = Request(Method.GET, loginFormUrl) + val response: Response = client(request) + + val csrfToken = htmlDocument(response.bodyString()) { + input { + withId = csrfTokenName + withAttribute = "name" to csrfTokenName + findFirst { attribute("value") } + } + } + + println("csrfToken: $csrfToken") + return Cookie(csrfTokenName, csrfToken) + } + + + private fun login(): Cookie { + + val loginRequest = Request(Method.POST, loginUrl) + .header("Referer", loginFormUrl) +// .cookie(csrfCookie) + .form("csrf_token", csrfCookie.value) + .form("username_or_email", appProps.factorioModPortalUsername) + .form("password", appProps.factorioModPortalPassword.use { it }) + .form("next_url", "/profile") + + val loginResponse = client(loginRequest) + + println(loginResponse.bodyString()) + + val sessionCookie = loginResponse.cookies() + .first { it.name == "session" && it.domain == ".factorio.com" } + + println("sessionCookie: $sessionCookie") + + return sessionCookie + } + + fun getModInfo(modName: String): JsonObject { + val response = client( + Request(Method.GET, "https://mods.factorio.com/api/mods/${modName}/full") + .cookie(csrfCookie) + .cookie(sessionCookie) + ) + return jsonMapper.parseToJsonElement(response.bodyString()).jsonObject + } + + fun fetchModUploadToken(modName: String): String { + val response = client( + Request(Method.GET, "https://mods.factorio.com/mod/${modName}/edit") + .cookie(csrfCookie) + .cookie(sessionCookie) + ) + + return response.bodyString() + } + + companion object { + private const val csrfTokenName = "csrf_token" + } + +} diff --git a/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt new file mode 100644 index 0000000..0bb1279 --- /dev/null +++ b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt @@ -0,0 +1,14 @@ +package dev.adamko.kafkatorio + +import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString + + +private const val modName = "kafkatorio-events" + + +fun main() { + val client = FactorioModPortalClient() +// val modInfo = client.getModInfo(modName) +// println(modInfo.asPrettyJsonString()) + println(client.fetchModUploadToken(modName)) +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 8d3ab2b..174121e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -14,6 +14,8 @@ include( ":modules:kt-rcon", ":modules:web-map", + + ":modules:factorio-mod-portal-client", ) enableFeaturePreview("VERSION_CATALOGS")