add Gradle task to upload the mod to the portal, and some gradle fixes

modules/events-mod/build.gradle.kts

@@ -1,3 +1,4 @@
+import dev.adamko.kafkatorio.factoriomod.FactorioModPublishTask
 import dev.adamko.kafkatorio.gradle.asConsumer
 import dev.adamko.kafkatorio.gradle.asProvider
 import dev.adamko.kafkatorio.gradle.dropDirectories
@@ -50,7 +51,7 @@ val tsSrcDir: Directory = layout.projectDirectory.dir("src/main/typescript")
 
 
 node {
-  nodeProjectDir.set(tsSrcDir)
+  nodeProjectDir.set(layout.projectDirectory)
 }
 
 
@@ -115,6 +116,15 @@ tasks.distZip {
 }
 
 
+val publishMod by tasks.registering(FactorioModPublishTask::class) {
+  distributionZip.set(tasks.distZip.flatMap { it.archiveFile })
+
+  val projectModName = project.extra.get("modName") as String
+  modName.set(projectModName)
+  modVersion.set(project.version.toString())
+}
+
+
 distributions {
   main {
 

modules/events-server/build.gradle.kts

@@ -15,7 +15,7 @@ description = """
 
 val projectId: String by project.extra
 
-val kafkaStateDir = layout.projectDirectory.dir("kafka-state")
+val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
 
 dependencies {
   implementation(platform(projects.modules.versionsPlatform))
@@ -64,7 +64,7 @@ application {
 //  )
 }
 
-tasks.withType<KotlinCompile> {
+tasks.withType<KotlinCompile>().configureEach {
   kotlinOptions.freeCompilerArgs += listOf(
     "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
     "-opt-in=kotlinx.coroutines.FlowPreview",

modules/factorio-mod-portal-client/build.gradle.kts

@@ -3,32 +3,41 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 plugins {
   dev.adamko.kafkatorio.lang.`kotlin-jvm`
   kotlin("plugin.serialization")
-  application
+  `java-gradle-plugin`
 }
 
 
 dependencies {
   implementation(platform(projects.modules.versionsPlatform))
 
-  implementation(projects.modules.eventsServer)
+//  implementation(gradleKotlinDsl())
 
   implementation(libs.kotlinx.serialization.core)
   implementation(libs.kotlinx.serialization.json)
+  implementation(libs.kotlinx.cli)
 
   implementation(libs.kotlinx.coroutines.core)
 
-  implementation(libs.bundles.http4k)
-  implementation(libs.http4k.format.kotlinx)
+//  implementation(libs.bundles.http4k)
+//  implementation(libs.http4k.format.kotlinx)
 
-  implementation(libs.skrapeit)
+//  implementation(libs.skrapeit)
+
+  implementation(libs.okio.core)
+
+  implementation(libs.semver)
+
+  implementation(libs.ktor.clientAuth)
+  implementation(libs.ktor.clientContentNegotiation)
+  implementation(libs.ktor.clientCore)
+  implementation(libs.ktor.clientEncoding)
+  implementation(libs.ktor.clientLogging)
+  implementation(libs.ktor.clientCIO)
+  implementation(libs.ktor.clientResources)
+  implementation(libs.ktor.serializationKotlinxJson)
 }
 
-//
-//application {
-//  mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
-//}
-
-tasks.withType<KotlinCompile> {
+tasks.withType<KotlinCompile>().configureEach {
   kotlinOptions.freeCompilerArgs += listOf(
     "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
     "-opt-in=kotlinx.coroutines.FlowPreview",

modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/ApplicationProperties.kt

@@ -1,48 +0,0 @@
-package dev.adamko.kafkatorio
-
-import dev.adamko.kafkatorio.processor.config.env
-import dev.adamko.kafkatorio.processor.config.fromEnvVarPath
-import dev.adamko.kafkatorio.processor.config.fromResourcesFile
-import java.time.Clock
-import org.http4k.cloudnative.env.Environment
-import org.http4k.cloudnative.env.EnvironmentKey
-import org.http4k.cloudnative.env.Secret
-import org.http4k.lens.nonEmptyString
-import org.http4k.lens.secret
-
-
-/**
- * Load properties, in descending priority order, from
- *
- * 1. Environment variables
- * 2. This app's env file (as defined by the environment variable [Lenses.APPLICATION_PROPERTIES_FILE_PATH])
- * # 3. A resources-dir yaml file
- *
- */
-internal class ApplicationProperties(
-  vararg environments: Environment = arrayOf(
-    Environment.env(),
-    Environment.fromEnvVarPath(),
-    Environment.fromResourcesFile(".secret.config.yml")
-  )
-) {
-
-  private val appProps: Environment =
-    environments.reduce { acc, environment -> acc.overrides(environment) }
-
-
-  fun clock(): Clock = Clock.systemUTC()
-
-
-  val factorioModPortalUsername: String
-    get() = EnvironmentKey
-      .nonEmptyString()
-      .required("factorio-mod-portal.username")[appProps]
-
-
-  val factorioModPortalPassword: Secret
-    get() = EnvironmentKey
-      .secret()
-      .required("factorio-mod-portal.password")[appProps]
-
-}

modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalClient.kt

@@ -1,107 +1,107 @@
 package dev.adamko.kafkatorio
 
-import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString
-import dev.adamko.kafkatorio.processor.serdes.jsonMapper
-import it.skrape.core.htmlDocument
-import it.skrape.selects.html5.input
-import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.jsonObject
-import org.http4k.client.OkHttp
-import org.http4k.core.HttpHandler
-import org.http4k.core.Method
-import org.http4k.core.Request
-import org.http4k.core.Response
-import org.http4k.core.body.form
-import org.http4k.core.cookie.Cookie
-import org.http4k.core.cookie.cookie
-import org.http4k.core.cookie.cookies
-import org.http4k.core.then
-import org.http4k.filter.ClientFilters
-
-
-/**
- * * https://github.com/shanemadden/factorio-mod-portal-publish/blob/master/entrypoint.sh
- * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/entrypoint.py
- * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/factorioModPortal.py
- * * https://github.com/justarandomgeek/vscode-factoriomod-debug/blob/master/src/ModPackageProvider.ts#L530
- *
- *
- */
-internal class FactorioModPortalClient(
-  private val appProps: ApplicationProperties = ApplicationProperties(),
-  private val loginFormUrl: String = "https://factorio.com/login?mods=1",
-  private val loginUrl: String = "https://factorio.com/login",
-) {
-
-  private val client: HttpHandler =
-    ClientFilters.FollowRedirects()
-      .then(OkHttp())
-
-  private val csrfCookie: Cookie by lazy { fetchCsrfToken() }
-  private val sessionCookie: Cookie by lazy { login() }
-
-
-  private fun fetchCsrfToken(): Cookie {
-
-    val request = Request(Method.GET, loginFormUrl)
-    val response: Response = client(request)
-
-    val csrfToken = htmlDocument(response.bodyString()) {
-      input {
-        withId = csrfTokenName
-        withAttribute = "name" to csrfTokenName
-        findFirst { attribute("value") }
-      }
-    }
-
-    println("csrfToken: $csrfToken")
-    return Cookie(csrfTokenName, csrfToken)
-  }
-
-
-  private fun login(): Cookie {
-
-    val loginRequest = Request(Method.POST, loginUrl)
-      .header("Referer", loginFormUrl)
-//      .cookie(csrfCookie)
-      .form("csrf_token", csrfCookie.value)
-      .form("username_or_email", appProps.factorioModPortalUsername)
-      .form("password", appProps.factorioModPortalPassword.use { it })
-      .form("next_url", "/profile")
-
-    val loginResponse = client(loginRequest)
-
-    println(loginResponse.bodyString())
-
-    val sessionCookie = loginResponse.cookies()
-      .first { it.name == "session" && it.domain == ".factorio.com" }
-
-    println("sessionCookie: $sessionCookie")
-
-    return sessionCookie
-  }
-
-  fun getModInfo(modName: String): JsonObject {
-    val response = client(
-      Request(Method.GET, "https://mods.factorio.com/api/mods/${modName}/full")
-        .cookie(csrfCookie)
-        .cookie(sessionCookie)
-    )
-    return jsonMapper.parseToJsonElement(response.bodyString()).jsonObject
-  }
-
-    fun fetchModUploadToken(modName: String): String {
-    val response = client(
-      Request(Method.GET, "https://mods.factorio.com/mod/${modName}/edit")
-        .cookie(csrfCookie)
-        .cookie(sessionCookie)
-    )
-
-    return response.bodyString()
-  }
-
-  companion object {
-    private const val csrfTokenName = "csrf_token"
-  }
-
-}
+//import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString
+//import dev.adamko.kafkatorio.processor.serdes.jsonMapper
+//import it.skrape.core.htmlDocument
+//import it.skrape.selects.html5.input
+//import kotlinx.serialization.json.JsonObject
+//import kotlinx.serialization.json.jsonObject
+//import org.http4k.client.OkHttp
+//import org.http4k.core.HttpHandler
+//import org.http4k.core.Method
+//import org.http4k.core.Request
+//import org.http4k.core.Response
+//import org.http4k.core.body.form
+//import org.http4k.core.cookie.Cookie
+//import org.http4k.core.cookie.cookie
+//import org.http4k.core.cookie.cookies
+//import org.http4k.core.then
+//import org.http4k.filter.ClientFilters
+//
+//
+///*
+// * * https://github.com/shanemadden/factorio-mod-portal-publish/blob/master/entrypoint.sh
+// * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/entrypoint.py
+// * * https://github.com/nicolas-lang/factorio-create-release-action/blob/master/factorioModPortal.py
+// * * https://github.com/justarandomgeek/vscode-factoriomod-debug/blob/master/src/ModPackageProvider.ts#L530
+// *
+// *
+// */
+//internal class FactorioModPortalClient(
+//  private val appProps: ApplicationProperties = ApplicationProperties(),
+//  private val loginFormUrl: String = "https://factorio.com/login?mods=1",
+//  private val loginUrl: String = "https://factorio.com/login",
+//) {
+//
+//  private val client: HttpHandler =
+//    ClientFilters.FollowRedirects()
+//      .then(OkHttp())
+//
+//  private val csrfCookie: Cookie by lazy { fetchCsrfToken() }
+//  private val sessionCookie: Cookie by lazy { login() }
+//
+//
+//  private fun fetchCsrfToken(): Cookie {
+//
+//    val request = Request(Method.GET, loginFormUrl)
+//    val response: Response = client(request)
+//
+//    val csrfToken = htmlDocument(response.bodyString()) {
+//      input {
+//        withId = csrfTokenName
+//        withAttribute = "name" to csrfTokenName
+//        findFirst { attribute("value") }
+//      }
+//    }
+//
+//    println("csrfToken: $csrfToken")
+//    return Cookie(csrfTokenName, csrfToken)
+//  }
+//
+//
+//  private fun login(): Cookie {
+//
+//    val loginRequest = Request(Method.POST, loginUrl)
+//      .header("Referer", loginFormUrl)
+////      .cookie(csrfCookie)
+//      .form("csrf_token", csrfCookie.value)
+//      .form("username_or_email", appProps.factorioModPortalUsername)
+//      .form("password", appProps.factorioModPortalPassword.use { it })
+//      .form("next_url", "/profile")
+//
+//    val loginResponse = client(loginRequest)
+//
+//    println(loginResponse.bodyString())
+//
+//    val sessionCookie = loginResponse.cookies()
+//      .first { it.name == "session" && it.domain == ".factorio.com" }
+//
+//    println("sessionCookie: $sessionCookie")
+//
+//    return sessionCookie
+//  }
+//
+//  fun getModInfo(modName: String): JsonObject {
+//    val response = client(
+//      Request(Method.GET, "https://mods.factorio.com/api/mods/${modName}/full")
+//        .cookie(csrfCookie)
+//        .cookie(sessionCookie)
+//    )
+//    return jsonMapper.parseToJsonElement(response.bodyString()).jsonObject
+//  }
+//
+//    fun fetchModUploadToken(modName: String): String {
+//    val response = client(
+//      Request(Method.GET, "https://mods.factorio.com/mod/${modName}/edit")
+//        .cookie(csrfCookie)
+//        .cookie(sessionCookie)
+//    )
+//
+//    return response.bodyString()
+//  }
+//
+//  companion object {
+//    private const val csrfTokenName = "csrf_token"
+//  }
+//
+//}

modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/FactorioModPortalPublishTask.kt

@@ -0,0 +1,83 @@
+package dev.adamko.kafkatorio
+
+//import javax.inject.Inject
+//import kotlinx.coroutines.runBlocking
+//import kotlinx.serialization.json.Json
+//import net.swiftzer.semver.SemVer
+//import okio.FileSystem
+//import okio.Path.Companion.toOkioPath
+//import okio.Path.Companion.toPath
+//import okio.openZip
+//import org.gradle.api.DefaultTask
+//import org.gradle.api.file.RegularFileProperty
+//import org.gradle.api.model.ObjectFactory
+//import org.gradle.api.provider.Property
+//import org.gradle.api.provider.ProviderFactory
+//import org.gradle.api.tasks.Input
+//import org.gradle.api.tasks.InputFile
+//import org.gradle.api.tasks.TaskAction
+//import org.gradle.kotlin.dsl.property
+//
+//
+//abstract class FactorioModPortalPublishTask @Inject constructor(
+//  objects: ObjectFactory,
+//  providers: ProviderFactory,
+//) : DefaultTask() {
+//
+//  @get:InputFile
+//  abstract val distributionZip: RegularFileProperty
+//
+//  @get:Input
+//  abstract val modVersion: Property<String>
+//
+//  @get:Input
+//  abstract val modName: Property<String>
+//
+//  @get:Input
+//  abstract val factorioVersion: Property<String>
+//
+//  /** Bearer API key */
+//  @get:Input
+//  val portalApiKey: Property<String> = objects.property<String>().convention(
+//    providers.gradleProperty("FACTORIO_MOD_PORTAL_API_KEY")
+//  )
+//
+//  @get:Input
+//  val portalUploadEndpoint: Property<String> = objects.property<String>()
+//    .convention("https://mods.factorio.com/api/v2/mods/releases/init_upload")
+//
+//
+//  @TaskAction
+//  fun deploy() = runBlocking {
+//    val modVersion = SemVer.parse(modVersion.get())
+//
+//    val portalUploadEndpoint = portalUploadEndpoint.get()
+//    val modName = modName.get()
+//    val portalApiKey = portalApiKey.get()
+//    val distributionZip = distributionZip.get().asFile
+//
+//  }
+//
+//  private fun validateInfoJson() {
+//    val modVersion = modVersion.get()
+//    val factorioVersion = factorioVersion.get()
+//
+//    val distributionZip = distributionZip.get().asFile.toOkioPath()
+//
+//    val zipContents = FileSystem.SYSTEM.openZip(distributionZip)
+//
+//    val infoJson = zipContents.read("info.json".toPath()) {
+//      generateSequence { readUtf8Line() }
+//    }.joinToString("\n")
+//
+//    val info: FactorioModInfo = Json.decodeFromString(FactorioModInfo.serializer(), infoJson)
+//
+//    require(info.version == modVersion) {
+//      "mod version mismatch, ${info.version} vs $modVersion "
+//    }
+//
+//    require(info.factorioVersion == factorioVersion) {
+//      "Factorio version mismatch, ${info.factorioVersion} vs $factorioVersion "
+//    }
+//  }
+//}

modules/factorio-mod-portal-client/src/main/kotlin/dev/adamko/kafkatorio/modUploader.kt

@@ -1,14 +1,40 @@
 package dev.adamko.kafkatorio
-
-import dev.adamko.kafkatorio.processor.serdes.KXS.asPrettyJsonString
-
-
-private const val modName = "kafkatorio-events"
-
-
-fun main() {
-  val client = FactorioModPortalClient()
-//  val modInfo = client.getModInfo(modName)
-//  println(modInfo.asPrettyJsonString())
-  println(client.fetchModUploadToken(modName))
-}
+//
+//import java.io.File
+//import kotlinx.cli.ArgParser
+//import kotlinx.cli.ArgType
+//import kotlinx.cli.required
+//import kotlinx.coroutines.runBlocking
+//
+//fun main(args: Array<String>) = runBlocking {
+//  val parser = ArgParser("example")
+//
+//  val distributionZip: File by parser.option(FileArg).required()
+//  val modName: String by parser.option(ArgType.String).required()
+//  val portalApiKey: String by parser.option(ArgType.String).required()
+//  val portalUploadEndpoint: String by parser.option(ArgType.String).required()
+//
+//  parser.parse(args)
+//
+//  val client = FactorioModPortalPublishClient(
+//    distributionZip,
+//    modName,
+//    portalApiKey,
+//    portalUploadEndpoint,
+//  )
+//
+//  client.uploadMod()
+//}
+//
+//
+//object FileArg : ArgType<File>(true) {
+//  override val description: kotlin.String
+//    get() = "{ File }"
+//
+//  override fun convert(value: kotlin.String, name: kotlin.String): File {
+//    return File(value).also {
+//      require(it.exists()) { "${it.canonicalPath} does not exist" }
+//      require(it.isFile) { "${it.canonicalPath} is not a file" }
+//    }
+//  }
+//}

modules/versions-platform/build.gradle.kts

@@ -24,18 +24,12 @@ dependencies {
   constraints {
     api(libs.skrapeit)
     api(libs.kotlinx.serialization.yaml)
-
-
-
-
-
-
-
-
     api(libs.kafka.kotkaStreams)
     api(libs.kafka.streams)
     api(libs.okio.core)
     api(libs.scrimage.core)
     api(libs.kafka.streamsTestUtils)
+
+    api(libs.mockk)
   }
 }

modules/web-map/build.gradle.kts

@@ -36,7 +36,10 @@ kotlin {
           port = 3000,
           proxy = mutableMapOf(
             "/kv/*" to "http://localhost:8080",
-            "/kvws/*" to mapOf("target" to "ws://localhost:8080", "ws" to true)
+            "/kvws/*" to mapOf(
+              "target" to "ws://localhost:8080",
+              "ws" to true,
+            )
           ),
           static = mutableListOf("$buildDir/processedResources/frontend/main")
         )
@@ -160,3 +163,6 @@ fun KotlinDependencyHandler.kvision(
 //    resolution("node-forge", "1.3.0")
 //  }
 //}
+
+evaluationDependsOn(rootProject.path)
+evaluationDependsOn(projects.modules.eventsSchema.dependencyProject.path)