factorio -> event -> mod -> console -> kafka pipe -> kafka -> event processor working!!!!!!!!!!!!!!!!!!

modules/events-mod/build.gradle.kts

@@ -52,7 +52,7 @@ val typescriptToLua by tasks.registering(NpmTask::class) {
   description = "Convert Typescript To Lua"
   group = project.name
 
-  dependsOn(tasks.npmInstall)
+  dependsOn(tasks.npmInstall, fetchEventsSchema)
 
   execOverrides { standardOutput = System.out }
 
@@ -121,12 +121,13 @@ distributions {
     contents {
       from(layout.projectDirectory.dir("src/main/resources/mod-data")) {
         include("**/**")
-        filter<ReplaceTokens>("tokens" to tokens)
       }
       from(licenseFile)
       from(typescriptToLua.map { it.outputs })
+      filter<ReplaceTokens>("tokens" to tokens)
       includeEmptyDirs = false
       exclude {
+        // exclude empty lines
         it.file.run {
           isFile && useLines { lines -> lines.all { line -> line.isBlank() } }
         }

modules/events-mod/src/main/typescript/events.ts

@@ -1,6 +1,7 @@
 import {Serdes} from "./serdes/serdes"
 
-const MOD_VERSION = script.active_mods["factorio-web-map"]
+const MOD_VERSION = script.active_mods["@mod.name@"] ?? "UNKNOWN"
+const FACTORIO_VERSION = script.active_mods["base"] ?? "UNKNOWN"
 
 const mapEventIdToName = new LuaTable<defines.Events, keyof typeof defines.events>()
 for (const [k, v] of pairs(defines.events)) {
@@ -48,6 +49,7 @@ function emitEvent<T extends FactorioObjectData>(eventData: T, tick: uint, event
     data: eventData,
     event_type: eventType,
     mod_version: MOD_VERSION,
+    factorio_version: FACTORIO_VERSION,
     tick: tick
   }
 

modules/events-mod/src/main/typescript/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.0.7",
+  "version": "0.1.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

modules/events-mod/src/main/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.0.7",
+  "version": "0.1.1",
   "private": true,
   "scripts": {
     "build": "tstl",
@@ -9,7 +9,8 @@
   "engines": {
     "node": ">=10.13.0"
   },
-  "devDependencies": {},
+  "devDependencies": {
+  },
   "dependencies": {
     "lua-types": "^2.11.0",
     "@types/node": "^12.0.8",
@@ -20,7 +21,10 @@
   "//": {
     "typed-factorio": "0.10.0"
   },
-  "peerDependencies": {},
-  "optionalDependencies": {},
-  "bundledDependencies": []
+  "peerDependencies": {
+  },
+  "optionalDependencies": {
+  },
+  "bundledDependencies": [
+  ]
 }

modules/events-mod/src/main/typescript/schema/events-schema.ts

@@ -8,6 +8,7 @@ interface FactorioObjectData {
 interface FactorioEvent<T extends FactorioObjectData> {
     data: T;
     event_type: string;
+    factorio_version: string;
     mod_version: string;
     tick: uint;
 }
@@ -21,7 +22,6 @@ interface PlayerData extends FactorioObjectData {
     associated_characters_unit_numbers: uint[];
     character_unit_number: uint | null;
     name: string;
-    object_name: string;
     position: PositionData;
 }
 
@@ -30,7 +30,6 @@ interface EntityData extends FactorioObjectData {
     health: double | null;
     health_ratio: double;
     name: string;
-    object_name: string;
     player_index: uint | null;
     position: PositionData;
     surface_index: int;
@@ -42,5 +41,4 @@ interface SurfaceData extends FactorioObjectData {
     daytime: double;
     index: uint;
     name: string;
-    object_name: string;
 }

modules/events-processor/src/main/kotlin/dev/adamko/kafkatorio/processor/main.kt

@@ -5,13 +5,40 @@ import dev.adamko.kafkatorio.events.schema.FactorioObjectData
 import dev.adamko.kafkatorio.events.schema.jsonMapper
 import dev.adamko.kafkatorio.processor.config.ApplicationProperties
 import java.time.Duration
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.modules.SerializersModule
+import org.apache.kafka.common.serialization.Deserializer
+import org.apache.kafka.common.serialization.Serde
 import org.apache.kafka.common.serialization.Serdes
+import org.apache.kafka.common.serialization.Serializer
 import org.apache.kafka.streams.KafkaStreams
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.TopologyDescription
 import org.apache.kafka.streams.kstream.Consumed
-import org.apache.kafka.streams.processor.TopicNameExtractor
-import org.http4k.format.KotlinxSerialization
+import org.apache.kafka.streams.kstream.Produced
+import org.http4k.format.ConfigurableKotlinxSerialization
+import org.http4k.format.asConfigurable
+import org.http4k.format.withStandardMappings
+
+object KSX : ConfigurableKotlinxSerialization({
+  ignoreUnknownKeys = true
+  prettyPrint = true
+  prettyPrintIndent = "  "
+  asConfigurable().withStandardMappings().done()
+  serializersModule = SerializersModule { }
+})
+
+object JsonSerdes : Serde<FactorioEvent<FactorioObjectData>> {
+  override fun serializer() = Serializer<FactorioEvent<FactorioObjectData>> { _, message ->
+    jsonMapper.encodeToString<FactorioEvent<FactorioObjectData>>(message).encodeToByteArray()
+  }
+
+  override fun deserializer() = Deserializer<FactorioEvent<FactorioObjectData>> { _, bytes ->
+    jsonMapper.decodeFromString(bytes.decodeToString())
+  }
+
+}
 
 fun main() {
 
@@ -25,16 +52,18 @@ fun main() {
   )
     .mapValues { readOnlyKey, value ->
       println("Mapping $readOnlyKey:$value")
-//      jsonMapper.decodeFromString<FactorioEvent<*>>(value)
-      KotlinxSerialization.asA<FactorioEvent<FactorioObjectData>>(value)
+      jsonMapper.decodeFromString<FactorioEvent<FactorioObjectData>>(value)
+//      KSX.asA<FactorioEvent<FactorioObjectData>>(value)
     }
-    .to(TopicNameExtractor { key, value, recordContext ->
-      value.data.objectName
-    })
+    .to(
+      { key, value, recordContext -> value.data.objectName },
+      Produced.with(Serdes.String(), JsonSerdes)
+    )
 
   val topology = builder.build()
   val properties = appProps.kafkaConfig
 
+
   val streams = KafkaStreams(topology, properties)
 
   streams.setUncaughtExceptionHandler(StreamsExceptionHandler())

modules/events-schema/build.gradle.kts

@@ -14,6 +14,8 @@ dependencies {
 
   implementation("com.github.ntrrgc:ts-generator:1.1.2")
   implementation("org.jetbrains.kotlin:kotlin-reflect")
+
+  testImplementation(libs.kotest.datatest)
 }
 
 val generateTypescript by tasks.registering(JavaExec::class) {

modules/events-schema/src/main/kotlin/dev/adamko/kafkatorio/events/schema/factorioEvents.kt

@@ -1,8 +1,11 @@
 package dev.adamko.kafkatorio.events.schema
 
 
+import kotlin.reflect.full.findAnnotation
+import kotlin.reflect.full.findAnnotations
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.Transient
 import kotlinx.serialization.json.JsonClassDiscriminator
 
 
@@ -15,6 +18,8 @@ data class FactorioEvent<out T : FactorioObjectData>(
   /** Schema versioning */
   @SerialName("mod_version")
   val modVersion: String,
+  @SerialName("factorio_version")
+  val factorioVersion: String,
   /** game time */
   @SerialName("tick")
   val tick: UInt,
@@ -29,17 +34,19 @@ const val FactorioObjectDataDiscriminatorKey: String = "object_name"
 
 @Serializable
 @JsonClassDiscriminator(FactorioObjectDataDiscriminatorKey)
-sealed class FactorioObjectData {
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  abstract val objectName: String
+sealed class FactorioObjectData private constructor() {
+
+  // workaround - https://github.com/Kotlin/kotlinx.serialization/issues/1664
+  val objectName : String by lazy {
+    requireNotNull(this::class.findAnnotation<SerialName>()?.value) {
+      "Couldn't find @SerialName for ${this::class}!"
+    }
+  }
 }
 
 @Serializable
 @SerialName("LuaPlayer")
 data class PlayerData(
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  override val objectName: String,
-
   @Serializable(with = ListAsObjectSerializer::class)
   @SerialName("associated_characters_unit_numbers")
   val associatedCharactersUnitNumbers: List<UInt>,
@@ -54,9 +61,6 @@ data class PlayerData(
 @Serializable
 @SerialName("LuaEntity")
 data class EntityData(
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  override val objectName: String,
-
   @SerialName("active")
   val active: Boolean,
   @SerialName("health")
@@ -72,7 +76,7 @@ data class EntityData(
   @SerialName("type")
   val type: String,
   @SerialName("unit_number")
-  val unitNumber: UInt?,
+  val unitNumber: UInt? = null,
 
   @SerialName("player_index")
   val playerIndex: UInt? = null,
@@ -81,9 +85,6 @@ data class EntityData(
 @Serializable
 @SerialName("LuaSurface")
 data class SurfaceData(
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  override val objectName: String,
-
   @SerialName("daytime")
   val daytime: Double,
   @SerialName("index")

modules/events-schema/src/test/kotlin/dev/adamko/kafkatorio/events/schema/FactorioEventTest.kt

@@ -1,14 +1,8 @@
 package dev.adamko.kafkatorio.events.schema
 
-import dev.adamko.kafkatorio.events.schema.EntityData
-import dev.adamko.kafkatorio.events.schema.FactorioEvent
-import dev.adamko.kafkatorio.events.schema.FactorioObjectData
-import dev.adamko.kafkatorio.events.schema.PlayerData
-import dev.adamko.kafkatorio.events.schema.PositionData
-import dev.adamko.kafkatorio.events.schema.SurfaceData
-import dev.adamko.kafkatorio.events.schema.jsonMapper
 import io.kotest.core.spec.style.BehaviorSpec
 import io.kotest.matchers.equality.shouldBeEqualToComparingFields
+import io.kotest.matchers.shouldBe
 import kotlinx.serialization.decodeFromString
 import org.intellij.lang.annotations.Language
 
@@ -31,6 +25,7 @@ class FactorioEventTest : BehaviorSpec({
         },
         "event_type": "on_player_joined_game",
         "mod_version": "0.0.4",
+        "factorio_version": "1.1.48",
         "tick": 1278458
       }
     """.trimIndent()
@@ -40,7 +35,6 @@ class FactorioEventTest : BehaviorSpec({
 
       val expected = FactorioEvent(
         data = PlayerData(
-          objectName = "LuaPlayer",
           name = "fredthedeadhead",
           characterUnitNumber = 1u,
           associatedCharactersUnitNumbers = emptyList(),
@@ -51,10 +45,13 @@ class FactorioEventTest : BehaviorSpec({
         ),
         eventType = "on_player_joined_game",
         modVersion = "0.0.4",
+        factorioVersion = "1.1.48",
         tick = 1278458u
       )
 
       actual shouldBeEqualToComparingFields expected
+
+      expected.data.objectName shouldBe "LuaPlayer"
     }
   }
 
@@ -70,6 +67,7 @@ class FactorioEventTest : BehaviorSpec({
         },
         "event_type": "on_tick",
         "mod_version": "0.0.4",
+        "factorio_version": "1.1.48",
         "tick": 1278480
       }
     """.trimIndent()
@@ -78,17 +76,19 @@ class FactorioEventTest : BehaviorSpec({
 
       val expected = FactorioEvent(
         data = SurfaceData(
-          objectName = "LuaSurface",
           name = "nauvis",
           index = 1u,
           daytime = 0.8390800000511436707029133685864508152008056640625
         ),
         eventType = "on_tick",
         modVersion = "0.0.4",
+        factorioVersion = "1.1.48",
         tick = 1278480u
       )
 
       actual shouldBeEqualToComparingFields expected
+
+      expected.data.objectName shouldBe "LuaSurface"
     }
   }
 
@@ -112,6 +112,7 @@ class FactorioEventTest : BehaviorSpec({
         },
         "event_type": "on_player_changed_position",
         "mod_version": "0.0.6",
+        "factorio_version": "1.1.48",
         "tick": 1384507
       }
     """.trimIndent()
@@ -120,7 +121,6 @@ class FactorioEventTest : BehaviorSpec({
 
       val expected = FactorioEvent(
         data = EntityData(
-          objectName = "LuaEntity",
           name = "character",
           type = "character",
           active = true,
@@ -136,10 +136,13 @@ class FactorioEventTest : BehaviorSpec({
         ),
         eventType = "on_player_changed_position",
         modVersion = "0.0.6",
+        factorioVersion = "1.1.48",
         tick = 1384507u
       )
 
       actual shouldBeEqualToComparingFields expected
+
+      expected.data.objectName shouldBe "LuaEntity"
     }
   }
 

modules/infra-factorio-client/build.gradle.kts

@@ -45,7 +45,7 @@ val clientLaunch by tasks.registering(Exec::class) {
   onlyIf(!isFactorioRunning())
 
   dependsOn(deployModToClient)
-  mustRunAfter(clientKill)
+  mustRunAfter(clientKill, ":modules:infra-factorio-server:processRun")
 
   commandLine = parseSpaceSeparatedArgs(
 //    """explorer "steam://rungameid/$factorioGameId// --mp-connect localhost/" """ // not working

modules/infra-factorio-server/build.gradle.kts

@@ -1,5 +1,6 @@
 import dev.adamko.kafkatorio.gradle.asConsumer
 import dev.adamko.kafkatorio.gradle.factorioModAttributes
+import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
 
 plugins {
   id("dev.adamko.kafkatorio.infra.docker-compose")
@@ -29,6 +30,10 @@ val deployModToServer by tasks.registering(Copy::class) {
   }
 }
 
+tasks.dockerDown {
+  commandLine = parseSpaceSeparatedArgs(""" docker-compose stop """)
+}
+
 tasks.dockerUp {
   dependsOn(
     deployModToServer,

modules/infra-kafka-cluster/src/docker-compose.yml

@@ -8,6 +8,7 @@ services:
     restart: "unless-stopped"
     build:
       dockerfile: "kafka.Dockerfile"
+      context: "."
       target: "kafka-kraft"
       args:
         KAFKA_DL_URL: "https://dlcdn.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz"

modules/infra-kafka-pipe/src/docker-compose.yml

@@ -6,6 +6,7 @@ services:
     container_name: "kafka-pipe"
     build:
       dockerfile: "kafka-pipe.Dockerfile"
+      context: "."
     restart: "unless-stopped"
     privileged: true
     networks: