diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts
index 9e7f59f..2d4696c 100644
--- a/buildSrc/build.gradle.kts
+++ b/buildSrc/build.gradle.kts
@@ -11,14 +11,16 @@ plugins {
object Versions {
const val jvmTarget = "11"
const val kotlinTarget = "1.6"
+ const val kotlin = "1.6.10"
const val gradleNodePlugin = "3.2.0"
const val gradleDockerComposePlugin = "0.15.0"
- const val kotlin = "1.6.10"
const val kotlinXSerialization = "1.3.2"
const val kvision = "5.8.1"
const val semver = "1.1.2"
+ const val http4k = "4.19.1.0"
+ const val jsoup = "1.14.3"
const val kotest = "5.1.0"
}
@@ -31,7 +33,7 @@ dependencies {
implementation("org.jetbrains.kotlin:kotlin-reflect")
implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:${Versions.kotlin}")
- implementation(enforcedPlatform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}"))
+ implementation(platform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}"))
implementation("org.jetbrains.kotlinx:kotlinx-serialization-core")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json")
// implementation("com.charleskorn.kaml:kaml:0.37.0")
@@ -46,6 +48,12 @@ dependencies {
// https://github.com/avast/gradle-docker-compose-plugin
implementation("com.avast.gradle:gradle-docker-compose-plugin:${Versions.gradleDockerComposePlugin}")
+
+ implementation(platform("org.http4k:http4k-bom:${Versions.http4k}"))
+ implementation("org.http4k:http4k-core")
+ implementation("org.http4k:http4k-client-okhttp")
+
+ implementation("org.jsoup:jsoup:${Versions.jsoup}")
}
diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts
index 8065bb4..f5a0553 100644
--- a/buildSrc/settings.gradle.kts
+++ b/buildSrc/settings.gradle.kts
@@ -1,3 +1,13 @@
+
rootProject.name = "buildSrc"
apply(from = "./repositories.settings.gradle.kts")
+
+@Suppress("UnstableApiUsage") // Central declaration of repositories is an incubating feature
+dependencyResolutionManagement {
+ versionCatalogs {
+ create("libs") {
+ from(files("../gradle/libs.versions.toml"))
+ }
+ }
+}
diff --git a/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/FactorioModPortalDeploy.kt b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/FactorioModPortalDeploy.kt
new file mode 100644
index 0000000..73ca284
--- /dev/null
+++ b/buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/FactorioModPortalDeploy.kt
@@ -0,0 +1,23 @@
+package dev.adamko.kafkatorio.task
+
+import org.gradle.api.DefaultTask
+import org.http4k.client.OkHttp
+import org.http4k.core.Method
+import org.http4k.core.Request
+
+abstract class FactorioModPortalDeploy : DefaultTask() {
+
+ init {
+ super.doFirst {
+
+ val client = OkHttp()
+
+ val request = Request(Method.GET, "https://factorio.com/login?mods=1")
+
+ println(client(request))
+
+
+ }
+ }
+
+}
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index a2ff04b..bd0d438 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -90,10 +90,12 @@ gcp-speech = { group = "com.google.cloud", name = "google-cloud-speech"
## kafka ##
kafka-streams = { group = "org.apache.kafka", name = "kafka-streams", version.ref = "kafka" }
-
# kafka helpers
kotka = { group = "com.github.blueanvil", name = "kotka", version = "1.1.1" }
+jsoup = { group = "org.jsoup", name = "jsoup", version = "1.14.3" }
+skrapeit = { group = "it.skrape", name = "skrapeit", version = "1.2.0" }
+
#############
### test ###
diff --git a/modules/events-server/build.gradle.kts b/modules/events-server/build.gradle.kts
index 8e140b2..be8c3c1 100644
--- a/modules/events-server/build.gradle.kts
+++ b/modules/events-server/build.gradle.kts
@@ -40,14 +40,9 @@ dependencies {
implementation("com.sksamuel.scrimage:scrimage-core:4.0.24")
val kotkaVersion = "v0.0.12"
-// val kotkaVersion = "store-iterators-SNAPSHOT"
implementation("com.github.adamko-dev.kotka-streams:kotka-streams:$kotkaVersion") {
isChanging = true
}
-// val kotkaVersion = "main-SNAPSHOT"
-// implementation("dev.adamko.kotka:kotka-streams:$kotkaVersion") {
-// isChanging = true
-// }
}
diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt
index c1784f6..3f578e1 100644
--- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt
+++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt
@@ -10,13 +10,14 @@ import org.apache.kafka.clients.admin.CreateTopicsResult
import org.apache.kafka.clients.admin.NewTopic
import org.apache.kafka.clients.admin.TopicListing
-private val logger = KotlinLogging.logger {}
//fun main() {
// KafkaAdmin.createKafkatorioTopics()
//}
-object KafkaAdmin {
+object KafkatorioKafkaAdmin {
+
+ private val logger = KotlinLogging.logger {}
private val kafkaAdmin: Admin =
Admin.create(
diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt
index efba5aa..26269b1 100644
--- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt
+++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt
@@ -5,6 +5,7 @@ import dev.adamko.kafkatorio.events.schema.FactorioEvent
import dev.adamko.kafkatorio.events.schema.FactorioObjectData
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kafkatorio.processor.topology.FactorioServerId
@@ -36,8 +37,9 @@ import org.apache.kafka.streams.kstream.KStream
import org.apache.kafka.streams.kstream.KTable
-class KafkatorioTopology(
+internal class KafkatorioTopology(
private val websocketServer: WebsocketServer,
+ private val appProps: ApplicationProperties = ApplicationProperties(),
) : CoroutineScope {
override val coroutineContext: CoroutineContext =
diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt
index a7f523e..e9ec6e3 100644
--- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt
+++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt
@@ -1,8 +1,10 @@
package dev.adamko.kafkatorio.processor.config
import dev.adamko.kafkatorio.processor.config.PortVal.Companion.portVal
+import java.nio.file.Path
import java.time.Clock
import java.util.Properties
+import kotlin.io.path.Path
import org.http4k.cloudnative.env.Environment
import org.http4k.cloudnative.env.EnvironmentKey
import org.http4k.lens.nonEmptyString
@@ -15,19 +17,21 @@ import org.http4k.lens.nonEmptyString
* # 3. A resources-dir yaml file
*
*/
-class ApplicationProperties(
+internal class ApplicationProperties(
vararg environments: Environment = arrayOf(
Environment.env(),
Environment.fromEnvVarPath(),
Environment.fromResourcesFile(".secret.config.yml")
)
) {
+
+
private val appProps: Environment =
environments.reduce { acc, environment -> acc.overrides(environment) }
+
fun clock(): Clock = Clock.systemUTC()
- //
/**
* The port of the server
@@ -37,10 +41,6 @@ class ApplicationProperties(
val serverPort: PortVal
get() = EnvironmentKey.portVal().defaulted("PORT", PortVal.DEFAULT)[appProps]
- //
-
-
- //
val kafkaConfig: Properties
get() = EnvironmentKey
@@ -54,6 +54,11 @@ class ApplicationProperties(
}
.required("kafka.config")[appProps]
- //
+
+ val webmapTileDir: Path
+ get() = EnvironmentKey
+ .nonEmptyString()
+ .map(::Path)
+ .required("tileServer.webmapTileDir")[appProps]
}
diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt
index 58d592e..4bfeec7 100644
--- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt
+++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt
@@ -13,11 +13,11 @@ import org.http4k.lens.Lens
*
* (An improved version of [Environment.fromResource])
*/
-internal fun Environment.Companion.fromResourcesFile(
+fun Environment.Companion.fromResourcesFile(
resource: String,
default: Environment = EMPTY
-) =
- try {
+): Environment {
+ return try {
val file =
this::class.java
.getResource("/${resource.removePrefix("/")}")
@@ -27,6 +27,7 @@ internal fun Environment.Companion.fromResourcesFile(
} catch (e: FileNotFoundException) {
default
}
+}
/**
* Try fetching a properties file the location provided by the
@@ -35,7 +36,7 @@ internal fun Environment.Companion.fromResourcesFile(
*
* If it does not exist, return [default].
*/
-internal fun Environment.Companion.fromEnvVarPath(
+fun Environment.Companion.fromEnvVarPath(
location: Lens = Lenses.APPLICATION_PROPERTIES_FILE_PATH,
default: Environment = EMPTY,
): Environment {
@@ -65,7 +66,7 @@ internal fun Environment.Companion.fromFile(
}
/** The same as [Environment.ENV], but refreshes on each invocation (which is useful for testing). */
-internal fun Environment.Companion.env(): Environment =
+fun Environment.Companion.env(): Environment =
MapEnvironment.from(System.getenv().toProperties())
/** The same as [Environment.JVM_PROPERTIES], but refreshes on each invocation (which is useful for testing). */
diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt
index 8e0c334..e53bf6d 100644
--- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt
+++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt
@@ -1,7 +1,7 @@
package dev.adamko.kafkatorio.processor
-import dev.adamko.kafkatorio.admin.KafkaAdmin
+import dev.adamko.kafkatorio.admin.KafkatorioKafkaAdmin
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.tileserver.WebMapTileServer
import kotlinx.coroutines.runBlocking
@@ -10,12 +10,12 @@ import org.http4k.server.Undertow
import org.http4k.server.asServer
-val appProps = ApplicationProperties()
+//private val appProps = ApplicationProperties()
fun main(): Unit = runBlocking {
- KafkaAdmin.createKafkatorioTopics()
+ KafkatorioKafkaAdmin.createKafkatorioTopics()
val wsServer = WebsocketServer()
val tileServer = WebMapTileServer()
diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt
index db5e420..db9f440 100644
--- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt
+++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt
@@ -1,5 +1,7 @@
package dev.adamko.kafkatorio.processor.tileserver
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import kotlin.io.path.absolutePathString
import org.http4k.core.HttpHandler
import org.http4k.core.then
import org.http4k.filter.CachingFilters
@@ -9,17 +11,13 @@ import org.http4k.routing.bind
import org.http4k.routing.routes
import org.http4k.routing.static
-class WebMapTileServer {
+internal class WebMapTileServer(
+ appProps: ApplicationProperties = ApplicationProperties()
+) {
private val routes: RoutingHttpHandler = routes(
"/tiles" bind CachingFilters.Response.NoCache()
- .then(
- static(
- ResourceLoader.Directory(
- """D:\Users\Adam\Projects\games\kafkatorio\modules\events-server\src\main\resources\kafkatorio-web-map\"""
- )
- )
- )
+ .then(static(ResourceLoader.Directory(appProps.webmapTileDir.absolutePathString())))
)
fun build(): HttpHandler {
diff --git a/modules/factorio-mod-portal-client/build.gradle.kts b/modules/factorio-mod-portal-client/build.gradle.kts
new file mode 100644
index 0000000..0f3cb8d
--- /dev/null
+++ b/modules/factorio-mod-portal-client/build.gradle.kts
@@ -0,0 +1,37 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+plugins {
+ dev.adamko.kafkatorio.lang.`kotlin-jvm`
+ kotlin("plugin.serialization")
+ application
+}
+
+
+dependencies {
+
+ implementation(projects.modules.eventsServer)
+
+ implementation(platform(libs.kotlinx.serialization.bom))
+ implementation(libs.kotlinx.serialization.core)
+ implementation(libs.kotlinx.serialization.json)
+
+ implementation(libs.kotlinx.coroutines)
+
+ implementation(platform(libs.http4k.bom))
+ implementation(libs.bundles.http4k)
+ implementation(libs.http4k.format.kotlinx)
+
+ implementation(libs.skrapeit)
+}
+
+//
+//application {
+// mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
+//}
+
+tasks.withType {
+ kotlinOptions.freeCompilerArgs += listOf(
+ "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
+ "-opt-in=kotlinx.coroutines.FlowPreview",
+ )
+}
diff --git a/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt
new file mode 100644
index 0000000..be38903
--- /dev/null
+++ b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt
@@ -0,0 +1,48 @@
+package dev.adamko.kafkatorio
+
+import dev.adamko.kafkatorio.processor.config.env
+import dev.adamko.kafkatorio.processor.config.fromEnvVarPath
+import dev.adamko.kafkatorio.processor.config.fromResourcesFile
+import java.time.Clock
+import org.http4k.cloudnative.env.Environment
+import org.http4k.cloudnative.env.EnvironmentKey
+import org.http4k.cloudnative.env.Secret
+import org.http4k.lens.nonEmptyString
+import org.http4k.lens.secret
+
+
+/**
+ * Load properties, in descending priority order, from
+ *
+ * 1. Environment variables
+ * 2. This app's env file (as defined by the environment variable [Lenses.APPLICATION_PROPERTIES_FILE_PATH])
+ * # 3. A resources-dir yaml file
+ *
+ */
+internal class ApplicationProperties(
+ vararg environments: Environment = arrayOf(
+ Environment.env(),
+ Environment.fromEnvVarPath(),
+ Environment.fromResourcesFile(".secret.config.yml")
+ )
+) {
+
+ private val appProps: Environment =
+ environments.reduce { acc, environment -> acc.overrides(environment) }
+
+
+ fun clock(): Clock = Clock.systemUTC()
+
+
+ val factorioModPortalUsername: String
+ get() = EnvironmentKey
+ .nonEmptyString()
+ .required("factorio-mod-portal.username")[appProps]
+
+
+ val factorioModPortalPassword: Secret
+ get() = EnvironmentKey
+ .secret()
+ .required("factorio-mod-portal.password")[appProps]
+
+}
diff --git a/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt
new file mode 100644
index 0000000..55fb5e2
--- /dev/null
+++ b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt
@@ -0,0 +1,107 @@
+package dev.adamko.kafkatorio
+
+import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString
+import dev.adamko.kafkatorio.processor.serdes.jsonMapper
+import it.skrape.core.htmlDocument
+import it.skrape.selects.html5.input
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.jsonObject
+import org.http4k.client.OkHttp
+import org.http4k.core.HttpHandler
+import org.http4k.core.Method
+import org.http4k.core.Request
+import org.http4k.core.Response
+import org.http4k.core.body.form
+import org.http4k.core.cookie.Cookie
+import org.http4k.core.cookie.cookie
+import org.http4k.core.cookie.cookies
+import org.http4k.core.then
+import org.http4k.filter.ClientFilters
+
+
+/**
+ * * https://github.com/shanemadden/factorio-mod-portal-publish/blob/master/entrypoint.sh
+ * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/entrypoint.py
+ * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/factorioModPortal.py
+ * * https://github.com/justarandomgeek/vscode-factoriomod-debug/blob/master/src/ModPackageProvider.ts#L530
+ *
+ *
+ */
+internal class FactorioModPortalClient(
+ private val appProps: ApplicationProperties = ApplicationProperties(),
+ private val loginFormUrl: String = "https://factorio.com/login?mods=1",
+ private val loginUrl: String = "https://factorio.com/login",
+) {
+
+ private val client: HttpHandler =
+ ClientFilters.FollowRedirects()
+ .then(OkHttp())
+
+ private val csrfCookie: Cookie by lazy { fetchCsrfToken() }
+ private val sessionCookie: Cookie by lazy { login() }
+
+
+ private fun fetchCsrfToken(): Cookie {
+
+ val request = Request(Method.GET, loginFormUrl)
+ val response: Response = client(request)
+
+ val csrfToken = htmlDocument(response.bodyString()) {
+ input {
+ withId = csrfTokenName
+ withAttribute = "name" to csrfTokenName
+ findFirst { attribute("value") }
+ }
+ }
+
+ println("csrfToken: $csrfToken")
+ return Cookie(csrfTokenName, csrfToken)
+ }
+
+
+ private fun login(): Cookie {
+
+ val loginRequest = Request(Method.POST, loginUrl)
+ .header("Referer", loginFormUrl)
+// .cookie(csrfCookie)
+ .form("csrf_token", csrfCookie.value)
+ .form("username_or_email", appProps.factorioModPortalUsername)
+ .form("password", appProps.factorioModPortalPassword.use { it })
+ .form("next_url", "/profile")
+
+ val loginResponse = client(loginRequest)
+
+ println(loginResponse.bodyString())
+
+ val sessionCookie = loginResponse.cookies()
+ .first { it.name == "session" && it.domain == ".factorio.com" }
+
+ println("sessionCookie: $sessionCookie")
+
+ return sessionCookie
+ }
+
+ fun getModInfo(modName: String): JsonObject {
+ val response = client(
+ Request(Method.GET, "https://mods.factorio.com/api/mods/${modName}/full")
+ .cookie(csrfCookie)
+ .cookie(sessionCookie)
+ )
+ return jsonMapper.parseToJsonElement(response.bodyString()).jsonObject
+ }
+
+ fun fetchModUploadToken(modName: String): String {
+ val response = client(
+ Request(Method.GET, "https://mods.factorio.com/mod/${modName}/edit")
+ .cookie(csrfCookie)
+ .cookie(sessionCookie)
+ )
+
+ return response.bodyString()
+ }
+
+ companion object {
+ private const val csrfTokenName = "csrf_token"
+ }
+
+}
diff --git a/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt
new file mode 100644
index 0000000..0bb1279
--- /dev/null
+++ b/modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt
@@ -0,0 +1,14 @@
+package dev.adamko.kafkatorio
+
+import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString
+
+
+private const val modName = "kafkatorio-events"
+
+
+fun main() {
+ val client = FactorioModPortalClient()
+// val modInfo = client.getModInfo(modName)
+// println(modInfo.asPrettyJsonString())
+ println(client.fetchModUploadToken(modName))
+}
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 8d3ab2b..174121e 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -14,6 +14,8 @@ include(
":modules:kt-rcon",
":modules:web-map",
+
+ ":modules:factorio-mod-portal-client",
)
enableFeaturePreview("VERSION_CATALOGS")