- try making a factorio mod portal api client, for auto releasing

- make webmapTileDir a configurable variable

buildSrc/build.gradle.kts

@@ -11,14 +11,16 @@ plugins {
 object Versions {
   const val jvmTarget = "11"
   const val kotlinTarget = "1.6"
+  const val kotlin = "1.6.10"
 
   const val gradleNodePlugin = "3.2.0"
   const val gradleDockerComposePlugin = "0.15.0"
 
-  const val kotlin = "1.6.10"
   const val kotlinXSerialization = "1.3.2"
   const val kvision = "5.8.1"
   const val semver = "1.1.2"
+  const val http4k = "4.19.1.0"
+  const val jsoup = "1.14.3"
 
   const val kotest = "5.1.0"
 }
@@ -31,7 +33,7 @@ dependencies {
   implementation("org.jetbrains.kotlin:kotlin-reflect")
   implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:${Versions.kotlin}")
 
-  implementation(enforcedPlatform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}"))
+  implementation(platform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}"))
   implementation("org.jetbrains.kotlinx:kotlinx-serialization-core")
   implementation("org.jetbrains.kotlinx:kotlinx-serialization-json")
 //  implementation("com.charleskorn.kaml:kaml:0.37.0")
@@ -46,6 +48,12 @@ dependencies {
 
   // https://github.com/avast/gradle-docker-compose-plugin
   implementation("com.avast.gradle:gradle-docker-compose-plugin:${Versions.gradleDockerComposePlugin}")
+
+  implementation(platform("org.http4k:http4k-bom:${Versions.http4k}"))
+  implementation("org.http4k:http4k-core")
+  implementation("org.http4k:http4k-client-okhttp")
+
+  implementation("org.jsoup:jsoup:${Versions.jsoup}")
 }
 
 

buildSrc/settings.gradle.kts

@@ -1,3 +1,13 @@
+
 rootProject.name = "buildSrc"
 
 apply(from = "./repositories.settings.gradle.kts")
+
+@Suppress("UnstableApiUsage") // Central declaration of repositories is an incubating feature
+dependencyResolutionManagement {
+  versionCatalogs {
+    create("libs") {
+      from(files("../gradle/libs.versions.toml"))
+    }
+  }
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/FactorioModPortalDeploy.kt

@@ -0,0 +1,23 @@
+package dev.adamko.kafkatorio.task
+
+import org.gradle.api.DefaultTask
+import org.http4k.client.OkHttp
+import org.http4k.core.Method
+import org.http4k.core.Request
+
+abstract class FactorioModPortalDeploy : DefaultTask() {
+
+  init {
+    super.doFirst {
+
+      val client = OkHttp()
+
+      val request = Request(Method.GET, "https://factorio.com/login?mods=1")
+
+      println(client(request))
+
+
+    }
+  }
+
+}

gradle/libs.versions.toml

@@ -90,10 +90,12 @@ gcp-speech = { group = "com.google.cloud",  name = "google-cloud-speech"
 
 ## kafka ##
 kafka-streams = { group = "org.apache.kafka",     name = "kafka-streams", version.ref = "kafka" }
-
 # kafka helpers
 kotka         = { group = "com.github.blueanvil", name = "kotka",         version =     "1.1.1" }
 
+jsoup = { group = "org.jsoup", name = "jsoup", version = "1.14.3" }
+skrapeit = { group = "it.skrape", name = "skrapeit", version = "1.2.0" }
+
 
 #############
 ### test  ###

modules/events-server/build.gradle.kts

@@ -40,14 +40,9 @@ dependencies {
   implementation("com.sksamuel.scrimage:scrimage-core:4.0.24")
 
   val kotkaVersion = "v0.0.12"
-//  val kotkaVersion = "store-iterators-SNAPSHOT"
   implementation("com.github.adamko-dev.kotka-streams:kotka-streams:$kotkaVersion") {
     isChanging = true
   }
-//  val kotkaVersion = "main-SNAPSHOT"
-//  implementation("dev.adamko.kotka:kotka-streams:$kotkaVersion") {
-//    isChanging = true
-//  }
 
 }
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt

@@ -10,13 +10,14 @@ import org.apache.kafka.clients.admin.CreateTopicsResult
 import org.apache.kafka.clients.admin.NewTopic
 import org.apache.kafka.clients.admin.TopicListing
 
-private val logger = KotlinLogging.logger {}
 
 //fun main() {
 //  KafkaAdmin.createKafkatorioTopics()
 //}
 
-object KafkaAdmin {
+object KafkatorioKafkaAdmin {
+
+  private val logger = KotlinLogging.logger {}
 
   private val kafkaAdmin: Admin =
     Admin.create(

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt

@@ -5,6 +5,7 @@ import dev.adamko.kafkatorio.events.schema.FactorioEvent
 import dev.adamko.kafkatorio.events.schema.FactorioObjectData
 import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
 import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
 import dev.adamko.kafkatorio.processor.serdes.jsonMapper
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
 import dev.adamko.kafkatorio.processor.topology.FactorioServerId
@@ -36,8 +37,9 @@ import org.apache.kafka.streams.kstream.KStream
 import org.apache.kafka.streams.kstream.KTable
 
 
-class KafkatorioTopology(
+internal class KafkatorioTopology(
   private val websocketServer: WebsocketServer,
+  private val appProps: ApplicationProperties = ApplicationProperties(),
 ) : CoroutineScope {
 
   override val coroutineContext: CoroutineContext =

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt

@@ -1,8 +1,10 @@
 package dev.adamko.kafkatorio.processor.config
 
 import dev.adamko.kafkatorio.processor.config.PortVal.Companion.portVal
+import java.nio.file.Path
 import java.time.Clock
 import java.util.Properties
+import kotlin.io.path.Path
 import org.http4k.cloudnative.env.Environment
 import org.http4k.cloudnative.env.EnvironmentKey
 import org.http4k.lens.nonEmptyString
@@ -15,19 +17,21 @@ import org.http4k.lens.nonEmptyString
  * # 3. A resources-dir yaml file
  *
  */
-class ApplicationProperties(
+internal class ApplicationProperties(
   vararg environments: Environment = arrayOf(
     Environment.env(),
     Environment.fromEnvVarPath(),
     Environment.fromResourcesFile(".secret.config.yml")
   )
 ) {
+
+
   private val appProps: Environment =
     environments.reduce { acc, environment -> acc.overrides(environment) }
 
+
   fun clock(): Clock = Clock.systemUTC()
 
-  //<editor-fold desc="Server properties">
 
   /**
    * The port of the server
@@ -37,10 +41,6 @@ class ApplicationProperties(
   val serverPort: PortVal
     get() = EnvironmentKey.portVal().defaulted("PORT", PortVal.DEFAULT)[appProps]
 
-  //</editor-fold>
-
-
-  //<editor-fold desc="Kafka properties">
 
   val kafkaConfig: Properties
     get() = EnvironmentKey
@@ -54,6 +54,11 @@ class ApplicationProperties(
       }
       .required("kafka.config")[appProps]
 
-  //</editor-fold>
+
+  val webmapTileDir: Path
+    get() = EnvironmentKey
+      .nonEmptyString()
+      .map(::Path)
+      .required("tileServer.webmapTileDir")[appProps]
 
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/environmentExt.kt

@@ -13,11 +13,11 @@ import org.http4k.lens.Lens
  *
  * (An improved version of [Environment.fromResource])
  */
-internal fun Environment.Companion.fromResourcesFile(
+fun Environment.Companion.fromResourcesFile(
   resource: String,
   default: Environment = EMPTY
-) =
+): Environment {
-  try {
+  return try {
     val file =
       this::class.java
         .getResource("/${resource.removePrefix("/")}")
@@ -27,6 +27,7 @@ internal fun Environment.Companion.fromResourcesFile(
   } catch (e: FileNotFoundException) {
     default
   }
+}
 
 /**
  * Try fetching a properties file the location provided by the
@@ -35,7 +36,7 @@ internal fun Environment.Companion.fromResourcesFile(
  *
  * If it does not exist, return [default].
  */
-internal fun Environment.Companion.fromEnvVarPath(
+fun Environment.Companion.fromEnvVarPath(
   location: Lens<Environment, File?> = Lenses.APPLICATION_PROPERTIES_FILE_PATH,
   default: Environment = EMPTY,
 ): Environment {
@@ -65,7 +66,7 @@ internal fun Environment.Companion.fromFile(
 }
 
 /** The same as [Environment.ENV], but refreshes on each invocation (which is useful for testing). */
-internal fun Environment.Companion.env(): Environment =
+fun Environment.Companion.env(): Environment =
   MapEnvironment.from(System.getenv().toProperties())
 
 /** The same as [Environment.JVM_PROPERTIES], but refreshes on each invocation (which is useful for testing). */

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt

@@ -1,7 +1,7 @@
 package dev.adamko.kafkatorio.processor
 
 
-import dev.adamko.kafkatorio.admin.KafkaAdmin
+import dev.adamko.kafkatorio.admin.KafkatorioKafkaAdmin
 import dev.adamko.kafkatorio.processor.config.ApplicationProperties
 import dev.adamko.kafkatorio.processor.tileserver.WebMapTileServer
 import kotlinx.coroutines.runBlocking
@@ -10,12 +10,12 @@ import org.http4k.server.Undertow
 import org.http4k.server.asServer
 
 
-val appProps = ApplicationProperties()
+//private val appProps = ApplicationProperties()
 
 
 fun main(): Unit = runBlocking {
 
-  KafkaAdmin.createKafkatorioTopics()
+  KafkatorioKafkaAdmin.createKafkatorioTopics()
 
   val wsServer = WebsocketServer()
   val tileServer = WebMapTileServer()

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt

@@ -1,5 +1,7 @@
 package dev.adamko.kafkatorio.processor.tileserver
 
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import kotlin.io.path.absolutePathString
 import org.http4k.core.HttpHandler
 import org.http4k.core.then
 import org.http4k.filter.CachingFilters
@@ -9,17 +11,13 @@ import org.http4k.routing.bind
 import org.http4k.routing.routes
 import org.http4k.routing.static
 
-class WebMapTileServer {
+internal class WebMapTileServer(
+  appProps: ApplicationProperties = ApplicationProperties()
+) {
 
   private val routes: RoutingHttpHandler = routes(
     "/tiles" bind CachingFilters.Response.NoCache()
-      .then(
+      .then(static(ResourceLoader.Directory(appProps.webmapTileDir.absolutePathString())))
-        static(
-          ResourceLoader.Directory(
-            """D:\Users\Adam\Projects\games\kafkatorio\modules\events-server\src\main\resources\kafkatorio-web-map\"""
-          )
-        )
-      )
   )
 
   fun build(): HttpHandler {

modules/factorio-mod-portal-client/build.gradle.kts

@@ -0,0 +1,37 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+plugins {
+  dev.adamko.kafkatorio.lang.`kotlin-jvm`
+  kotlin("plugin.serialization")
+  application
+}
+
+
+dependencies {
+
+  implementation(projects.modules.eventsServer)
+
+  implementation(platform(libs.kotlinx.serialization.bom))
+  implementation(libs.kotlinx.serialization.core)
+  implementation(libs.kotlinx.serialization.json)
+
+  implementation(libs.kotlinx.coroutines)
+
+  implementation(platform(libs.http4k.bom))
+  implementation(libs.bundles.http4k)
+  implementation(libs.http4k.format.kotlinx)
+
+  implementation(libs.skrapeit)
+}
+
+//
+//application {
+//  mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
+//}
+
+tasks.withType<KotlinCompile> {
+  kotlinOptions.freeCompilerArgs += listOf(
+    "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
+    "-opt-in=kotlinx.coroutines.FlowPreview",
+  )
+}

modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt

@@ -0,0 +1,48 @@
+package dev.adamko.kafkatorio
+
+import dev.adamko.kafkatorio.processor.config.env
+import dev.adamko.kafkatorio.processor.config.fromEnvVarPath
+import dev.adamko.kafkatorio.processor.config.fromResourcesFile
+import java.time.Clock
+import org.http4k.cloudnative.env.Environment
+import org.http4k.cloudnative.env.EnvironmentKey
+import org.http4k.cloudnative.env.Secret
+import org.http4k.lens.nonEmptyString
+import org.http4k.lens.secret
+
+
+/**
+ * Load properties, in descending priority order, from
+ *
+ * 1. Environment variables
+ * 2. This app's env file (as defined by the environment variable [Lenses.APPLICATION_PROPERTIES_FILE_PATH])
+ * # 3. A resources-dir yaml file
+ *
+ */
+internal class ApplicationProperties(
+  vararg environments: Environment = arrayOf(
+    Environment.env(),
+    Environment.fromEnvVarPath(),
+    Environment.fromResourcesFile(".secret.config.yml")
+  )
+) {
+
+  private val appProps: Environment =
+    environments.reduce { acc, environment -> acc.overrides(environment) }
+
+
+  fun clock(): Clock = Clock.systemUTC()
+
+
+  val factorioModPortalUsername: String
+    get() = EnvironmentKey
+      .nonEmptyString()
+      .required("factorio-mod-portal.username")[appProps]
+
+
+  val factorioModPortalPassword: Secret
+    get() = EnvironmentKey
+      .secret()
+      .required("factorio-mod-portal.password")[appProps]
+
+}

modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt

@@ -0,0 +1,107 @@
+package dev.adamko.kafkatorio
+
+import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString
+import dev.adamko.kafkatorio.processor.serdes.jsonMapper
+import it.skrape.core.htmlDocument
+import it.skrape.selects.html5.input
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.jsonObject
+import org.http4k.client.OkHttp
+import org.http4k.core.HttpHandler
+import org.http4k.core.Method
+import org.http4k.core.Request
+import org.http4k.core.Response
+import org.http4k.core.body.form
+import org.http4k.core.cookie.Cookie
+import org.http4k.core.cookie.cookie
+import org.http4k.core.cookie.cookies
+import org.http4k.core.then
+import org.http4k.filter.ClientFilters
+
+
+/**
+ * * https://github.com/shanemadden/factorio-mod-portal-publish/blob/master/entrypoint.sh
+ * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/entrypoint.py
+ * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/factorioModPortal.py
+ * * https://github.com/justarandomgeek/vscode-factoriomod-debug/blob/master/src/ModPackageProvider.ts#L530
+ *
+ *
+ */
+internal class FactorioModPortalClient(
+  private val appProps: ApplicationProperties = ApplicationProperties(),
+  private val loginFormUrl: String = "https://factorio.com/login?mods=1",
+  private val loginUrl: String = "https://factorio.com/login",
+) {
+
+  private val client: HttpHandler =
+    ClientFilters.FollowRedirects()
+      .then(OkHttp())
+
+  private val csrfCookie: Cookie by lazy { fetchCsrfToken() }
+  private val sessionCookie: Cookie by lazy { login() }
+
+
+  private fun fetchCsrfToken(): Cookie {
+
+    val request = Request(Method.GET, loginFormUrl)
+    val response: Response = client(request)
+
+    val csrfToken = htmlDocument(response.bodyString()) {
+      input {
+        withId = csrfTokenName
+        withAttribute = "name" to csrfTokenName
+        findFirst { attribute("value") }
+      }
+    }
+
+    println("csrfToken: $csrfToken")
+    return Cookie(csrfTokenName, csrfToken)
+  }
+
+
+  private fun login(): Cookie {
+
+    val loginRequest = Request(Method.POST, loginUrl)
+      .header("Referer", loginFormUrl)
+//      .cookie(csrfCookie)
+      .form("csrf_token", csrfCookie.value)
+      .form("username_or_email", appProps.factorioModPortalUsername)
+      .form("password", appProps.factorioModPortalPassword.use { it })
+      .form("next_url", "/profile")
+
+    val loginResponse = client(loginRequest)
+
+    println(loginResponse.bodyString())
+
+    val sessionCookie = loginResponse.cookies()
+      .first { it.name == "session" && it.domain == ".factorio.com" }
+
+    println("sessionCookie: $sessionCookie")
+
+    return sessionCookie
+  }
+
+  fun getModInfo(modName: String): JsonObject {
+    val response = client(
+      Request(Method.GET, "https://mods.factorio.com/api/mods/${modName}/full")
+        .cookie(csrfCookie)
+        .cookie(sessionCookie)
+    )
+    return jsonMapper.parseToJsonElement(response.bodyString()).jsonObject
+  }
+
+    fun fetchModUploadToken(modName: String): String {
+    val response = client(
+      Request(Method.GET, "https://mods.factorio.com/mod/${modName}/edit")
+        .cookie(csrfCookie)
+        .cookie(sessionCookie)
+    )
+
+    return response.bodyString()
+  }
+
+  companion object {
+    private const val csrfTokenName = "csrf_token"
+  }
+
+}

modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt

@@ -0,0 +1,14 @@
+package dev.adamko.kafkatorio
+
+import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString
+
+
+private const val modName = "kafkatorio-events"
+
+
+fun main() {
+  val client = FactorioModPortalClient()
+//  val modInfo = client.getModInfo(modName)
+//  println(modInfo.asPrettyJsonString())
+  println(client.fetchModUploadToken(modName))
+}

settings.gradle.kts

@@ -14,6 +14,8 @@ include(
 
   ":modules:kt-rcon",
   ":modules:web-map",
+
+  ":modules:factorio-mod-portal-client",
 )
 
 enableFeaturePreview("VERSION_CATALOGS")