factorio -> event -> mod -> console -> kafka pipe -> kafka -> event processor working!!!!!!!!!!!!!!!!!!

README.md

@@ -9,11 +9,11 @@
 ## todo
 
 * add settings
-  * is active - whether files are exported or not
-  * delete files - clean up all files
-  * tick-rate - how often reports are generated (
-    * requires on some sort of in-mod cache of map<entity-id, event>)
-    * also divide by game-speed? to keep ratio the same
+    * is active - whether files are exported or not
+    * delete files - clean up all files
+    * tick-rate - how often reports are generated (
+        * requires on some sort of in-mod cache of map<entity-id, event>)
+        * also divide by game-speed? to keep ratio the same
 
 ### to investigate:
 
@@ -21,6 +21,11 @@
 * https://github.com/grafi-tt/lunajson
 * http://lua-users.org/wiki/CompressionAndArchiving
 * https://luapower.com/zlib
+* https://github.com/appgurueu/modlib#bluon
+* https://github.com/dingyi222666/lua-table-serialize
+* https://github.com/uriid1/lua-serialize
+* https://github.com/rxi/lume#lumeserializex
+* https://github.com/SwadicalRag/bON
 
 * https://square.github.io/okio/#communicate-on-a-socket-javakotlin
 
@@ -30,6 +35,16 @@
 
 * https://jbrandhorst.com/post/go-protobuf-tips/
 
+##### flatbuffers
+
+https://github.com/dvidelabs/flatcc
+
+
+##### z85 encoding
+
+* Kotlin https://gist.github.com/ischumacher/b4929f26341ebef62cbce35c65543eda
+* Lua https://web-eworks.github.io/lZ85/
+
 #### CBOR
 
 https://www.zash.se/lua-cbor.html

build.gradle.kts

@@ -6,7 +6,7 @@ plugins {
 }
 
 group = "dev.adamko"
-version = "0.0.7"
+version = "0.1.1"
 
 tasks.wrapper {
   gradleVersion = "7.3"
@@ -26,7 +26,7 @@ val runKafkatorio by tasks.registering {
   group = project.name
 
   dependsOn(
-    ":modules:infra-factorio-client:processRun",
-    ":modules:infra-factorio-server:processRun",
+    ":modules:infra-factorio-client:processRestart",
+    ":modules:infra-factorio-server:processRestart",
   )
 }

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/common.kt

@@ -0,0 +1,9 @@
+package dev.adamko.kafkatorio
+
+import kotlinx.serialization.json.Json
+
+
+val jsonMapper = Json {
+  prettyPrint = true
+  prettyPrintIndent = "  "
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/gradle/extensions.kt

@@ -1,8 +1,13 @@
 package dev.adamko.kafkatorio.gradle
 
+import dev.adamko.kafkatorio.jsonMapper
 import java.io.ByteArrayOutputStream
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.logging.LogLevel
 import org.gradle.api.specs.Spec
 import org.gradle.kotlin.dsl.invoke
 
@@ -12,10 +17,28 @@ fun Project.isProcessRunning(process: String, ignoreCase: Boolean = true): Spec<
   Spec<Task> {
     ByteArrayOutputStream().use { outputStream ->
       project.exec {
-        commandLine("tasklist")
+        commandLine("tasklist") // Windows only for now...
         standardOutput = outputStream
       }
       outputStream.toString()
         .contains(process, ignoreCase)
     }
-  }
+  }
+
+fun Project.areJsonPropertiesUpToDate(
+  packageJsonFile: RegularFileProperty,
+  properties: () -> Map<String, String>
+): Spec<Task> = Spec<Task> {
+  val packageJsonContent = packageJsonFile.get().asFile.readText()
+  val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
+
+  properties().all { (key, expectedVal) ->
+    val actualVal = packageJson[key]?.jsonPrimitive?.content
+    (expectedVal == actualVal).also {
+      logger.log(
+        if (it) LogLevel.LIFECYCLE else LogLevel.INFO,
+        "package.json has outdated property '$key' (expected: $expectedVal, actual: $actualVal)"
+      )
+    }
+  }
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/node.gradle.kts

@@ -1,20 +1,17 @@
 package dev.adamko.kafkatorio.lang
 
 import com.github.gradle.node.NodePlugin
-import dev.adamko.kafkatorio.task.UpdatePackageJson
+import dev.adamko.kafkatorio.gradle.areJsonPropertiesUpToDate
 import dev.adamko.kafkatorio.gradle.not
+import dev.adamko.kafkatorio.jsonMapper
+import dev.adamko.kafkatorio.task.UpdatePackageJson
 import kotlinx.serialization.encodeToString
-import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.JsonPrimitive
 import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
 import org.gradle.kotlin.dsl.support.useToRun
-import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin
-import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin
 import org.jetbrains.kotlin.util.suffixIfNot
 
-
 plugins {
   id("com.github.node-gradle.node")
   id("dev.adamko.kafkatorio.base")
@@ -27,32 +24,6 @@ node {
   distBaseUrl.set(null as String?) // set by dependencyResolutionManagement
 }
 
-project.plugins.withType<KotlinJsPlugin>()
-project.plugins.withType<KotlinMultiplatformPlugin>()
-
-
-val jsonMapper = Json {
-  prettyPrint = true
-  prettyPrintIndent = "  "
-}
-
-
-fun arePropertiesUpToDate(
-  packageJsonFile: RegularFileProperty,
-  properties: () -> Map<String, String>
-): Spec<Task> = Spec<Task> {
-  val packageJsonContent = packageJsonFile.get().asFile.readText()
-  val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
-
-  properties().all { (key, expectedVal) ->
-    val actualVal = packageJson[key]?.jsonPrimitive?.content
-    (expectedVal == actualVal).also {
-      if (!it)
-        logger.lifecycle("package.json has outdated property '$key' (expected: $expectedVal, actual: $actualVal)")
-    }
-  }
-}
-
 val updatePackageJson by tasks.registering(UpdatePackageJson::class) {
   group = NodePlugin.NODE_GROUP
   description = "Read the package.json file and update the version and name, based on the project."
@@ -60,7 +31,7 @@ val updatePackageJson by tasks.registering(UpdatePackageJson::class) {
   propertiesToCheck["version"] = "${project.version}"
 
   // check to see if the properties are already up-to-date
-  onlyIf(!arePropertiesUpToDate(packageJsonFile) { propertiesToCheck })
+  onlyIf(!areJsonPropertiesUpToDate(packageJsonFile) { propertiesToCheck })
 
   doLast {
     val packageJsonContent = packageJsonFile.get().asFile.readText()
@@ -74,7 +45,7 @@ val updatePackageJson by tasks.registering(UpdatePackageJson::class) {
         .encodeToString(packageJsonUpdate)
         .suffixIfNot("\n")
 
-    logger.lifecycle(packageJsonContentUpdated)
+    logger.debug(packageJsonContentUpdated)
 
     packageJsonFile.get().asFile.writer().useToRun {
       write(packageJsonContentUpdated)

modules/events-mod/build.gradle.kts

@@ -52,7 +52,7 @@ val typescriptToLua by tasks.registering(NpmTask::class) {
   description = "Convert Typescript To Lua"
   group = project.name
 
-  dependsOn(tasks.npmInstall)
+  dependsOn(tasks.npmInstall, fetchEventsSchema)
 
   execOverrides { standardOutput = System.out }
 
@@ -121,12 +121,13 @@ distributions {
     contents {
       from(layout.projectDirectory.dir("src/main/resources/mod-data")) {
         include("**/**")
-        filter<ReplaceTokens>("tokens" to tokens)
       }
       from(licenseFile)
       from(typescriptToLua.map { it.outputs })
+      filter<ReplaceTokens>("tokens" to tokens)
       includeEmptyDirs = false
       exclude {
+        // exclude empty lines
         it.file.run {
           isFile && useLines { lines -> lines.all { line -> line.isBlank() } }
         }

modules/events-mod/src/main/typescript/events.ts

@@ -1,6 +1,7 @@
 import {Serdes} from "./serdes/serdes"
 
-const MOD_VERSION = script.active_mods["factorio-web-map"]
+const MOD_VERSION = script.active_mods["@mod.name@"] ?? "UNKNOWN"
+const FACTORIO_VERSION = script.active_mods["base"] ?? "UNKNOWN"
 
 const mapEventIdToName = new LuaTable<defines.Events, keyof typeof defines.events>()
 for (const [k, v] of pairs(defines.events)) {
@@ -48,6 +49,7 @@ function emitEvent<T extends FactorioObjectData>(eventData: T, tick: uint, event
     data: eventData,
     event_type: eventType,
     mod_version: MOD_VERSION,
+    factorio_version: FACTORIO_VERSION,
     tick: tick
   }
 

modules/events-mod/src/main/typescript/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.0.7",
+  "version": "0.1.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

modules/events-mod/src/main/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.0.7",
+  "version": "0.1.1",
   "private": true,
   "scripts": {
     "build": "tstl",
@@ -9,7 +9,8 @@
   "engines": {
     "node": ">=10.13.0"
   },
-  "devDependencies": {},
+  "devDependencies": {
+  },
   "dependencies": {
     "lua-types": "^2.11.0",
     "@types/node": "^12.0.8",
@@ -20,7 +21,10 @@
   "//": {
     "typed-factorio": "0.10.0"
   },
-  "peerDependencies": {},
-  "optionalDependencies": {},
-  "bundledDependencies": []
+  "peerDependencies": {
+  },
+  "optionalDependencies": {
+  },
+  "bundledDependencies": [
+  ]
 }

modules/events-mod/src/main/typescript/schema/events-schema.ts

@@ -8,6 +8,7 @@ interface FactorioObjectData {
 interface FactorioEvent<T extends FactorioObjectData> {
     data: T;
     event_type: string;
+    factorio_version: string;
     mod_version: string;
     tick: uint;
 }
@@ -21,7 +22,6 @@ interface PlayerData extends FactorioObjectData {
     associated_characters_unit_numbers: uint[];
     character_unit_number: uint | null;
     name: string;
-    object_name: string;
     position: PositionData;
 }
 
@@ -30,7 +30,6 @@ interface EntityData extends FactorioObjectData {
     health: double | null;
     health_ratio: double;
     name: string;
-    object_name: string;
     player_index: uint | null;
     position: PositionData;
     surface_index: int;
@@ -42,5 +41,4 @@ interface SurfaceData extends FactorioObjectData {
     daytime: double;
     index: uint;
     name: string;
-    object_name: string;
 }

modules/events-processor/src/main/kotlin/dev/adamko/kafkatorio/processor/main.kt

@@ -5,13 +5,40 @@ import dev.adamko.kafkatorio.events.schema.FactorioObjectData
 import dev.adamko.kafkatorio.events.schema.jsonMapper
 import dev.adamko.kafkatorio.processor.config.ApplicationProperties
 import java.time.Duration
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.modules.SerializersModule
+import org.apache.kafka.common.serialization.Deserializer
+import org.apache.kafka.common.serialization.Serde
 import org.apache.kafka.common.serialization.Serdes
+import org.apache.kafka.common.serialization.Serializer
 import org.apache.kafka.streams.KafkaStreams
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.TopologyDescription
 import org.apache.kafka.streams.kstream.Consumed
-import org.apache.kafka.streams.processor.TopicNameExtractor
-import org.http4k.format.KotlinxSerialization
+import org.apache.kafka.streams.kstream.Produced
+import org.http4k.format.ConfigurableKotlinxSerialization
+import org.http4k.format.asConfigurable
+import org.http4k.format.withStandardMappings
+
+object KSX : ConfigurableKotlinxSerialization({
+  ignoreUnknownKeys = true
+  prettyPrint = true
+  prettyPrintIndent = "  "
+  asConfigurable().withStandardMappings().done()
+  serializersModule = SerializersModule { }
+})
+
+object JsonSerdes : Serde<FactorioEvent<FactorioObjectData>> {
+  override fun serializer() = Serializer<FactorioEvent<FactorioObjectData>> { _, message ->
+    jsonMapper.encodeToString<FactorioEvent<FactorioObjectData>>(message).encodeToByteArray()
+  }
+
+  override fun deserializer() = Deserializer<FactorioEvent<FactorioObjectData>> { _, bytes ->
+    jsonMapper.decodeFromString(bytes.decodeToString())
+  }
+
+}
 
 fun main() {
 
@@ -25,16 +52,18 @@ fun main() {
   )
     .mapValues { readOnlyKey, value ->
       println("Mapping $readOnlyKey:$value")
-//      jsonMapper.decodeFromString<FactorioEvent<*>>(value)
-      KotlinxSerialization.asA<FactorioEvent<FactorioObjectData>>(value)
+      jsonMapper.decodeFromString<FactorioEvent<FactorioObjectData>>(value)
+//      KSX.asA<FactorioEvent<FactorioObjectData>>(value)
     }
-    .to(TopicNameExtractor { key, value, recordContext ->
-      value.data.objectName
-    })
+    .to(
+      { key, value, recordContext -> value.data.objectName },
+      Produced.with(Serdes.String(), JsonSerdes)
+    )
 
   val topology = builder.build()
   val properties = appProps.kafkaConfig
 
+
   val streams = KafkaStreams(topology, properties)
 
   streams.setUncaughtExceptionHandler(StreamsExceptionHandler())

modules/events-schema/build.gradle.kts

@@ -14,6 +14,8 @@ dependencies {
 
   implementation("com.github.ntrrgc:ts-generator:1.1.2")
   implementation("org.jetbrains.kotlin:kotlin-reflect")
+
+  testImplementation(libs.kotest.datatest)
 }
 
 val generateTypescript by tasks.registering(JavaExec::class) {

modules/events-schema/src/main/kotlin/dev/adamko/kafkatorio/events/schema/factorioEvents.kt

@@ -1,8 +1,11 @@
 package dev.adamko.kafkatorio.events.schema
 
 
+import kotlin.reflect.full.findAnnotation
+import kotlin.reflect.full.findAnnotations
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
+import kotlinx.serialization.Transient
 import kotlinx.serialization.json.JsonClassDiscriminator
 
 
@@ -15,6 +18,8 @@ data class FactorioEvent<out T : FactorioObjectData>(
   /** Schema versioning */
   @SerialName("mod_version")
   val modVersion: String,
+  @SerialName("factorio_version")
+  val factorioVersion: String,
   /** game time */
   @SerialName("tick")
   val tick: UInt,
@@ -29,17 +34,19 @@ const val FactorioObjectDataDiscriminatorKey: String = "object_name"
 
 @Serializable
 @JsonClassDiscriminator(FactorioObjectDataDiscriminatorKey)
-sealed class FactorioObjectData {
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  abstract val objectName: String
+sealed class FactorioObjectData private constructor() {
+
+  // workaround - https://github.com/Kotlin/kotlinx.serialization/issues/1664
+  val objectName : String by lazy {
+    requireNotNull(this::class.findAnnotation<SerialName>()?.value) {
+      "Couldn't find @SerialName for ${this::class}!"
+    }
+  }
 }
 
 @Serializable
 @SerialName("LuaPlayer")
 data class PlayerData(
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  override val objectName: String,
-
   @Serializable(with = ListAsObjectSerializer::class)
   @SerialName("associated_characters_unit_numbers")
   val associatedCharactersUnitNumbers: List<UInt>,
@@ -54,9 +61,6 @@ data class PlayerData(
 @Serializable
 @SerialName("LuaEntity")
 data class EntityData(
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  override val objectName: String,
-
   @SerialName("active")
   val active: Boolean,
   @SerialName("health")
@@ -72,7 +76,7 @@ data class EntityData(
   @SerialName("type")
   val type: String,
   @SerialName("unit_number")
-  val unitNumber: UInt?,
+  val unitNumber: UInt? = null,
 
   @SerialName("player_index")
   val playerIndex: UInt? = null,
@@ -81,9 +85,6 @@ data class EntityData(
 @Serializable
 @SerialName("LuaSurface")
 data class SurfaceData(
-  @SerialName(FactorioObjectDataDiscriminatorKey)
-  override val objectName: String,
-
   @SerialName("daytime")
   val daytime: Double,
   @SerialName("index")

modules/events-schema/src/test/kotlin/dev/adamko/kafkatorio/events/schema/FactorioEventTest.kt

@@ -1,14 +1,8 @@
 package dev.adamko.kafkatorio.events.schema
 
-import dev.adamko.kafkatorio.events.schema.EntityData
-import dev.adamko.kafkatorio.events.schema.FactorioEvent
-import dev.adamko.kafkatorio.events.schema.FactorioObjectData
-import dev.adamko.kafkatorio.events.schema.PlayerData
-import dev.adamko.kafkatorio.events.schema.PositionData
-import dev.adamko.kafkatorio.events.schema.SurfaceData
-import dev.adamko.kafkatorio.events.schema.jsonMapper
 import io.kotest.core.spec.style.BehaviorSpec
 import io.kotest.matchers.equality.shouldBeEqualToComparingFields
+import io.kotest.matchers.shouldBe
 import kotlinx.serialization.decodeFromString
 import org.intellij.lang.annotations.Language
 
@@ -31,6 +25,7 @@ class FactorioEventTest : BehaviorSpec({
         },
         "event_type": "on_player_joined_game",
         "mod_version": "0.0.4",
+        "factorio_version": "1.1.48",
         "tick": 1278458
       }
     """.trimIndent()
@@ -40,7 +35,6 @@ class FactorioEventTest : BehaviorSpec({
 
       val expected = FactorioEvent(
         data = PlayerData(
-          objectName = "LuaPlayer",
           name = "fredthedeadhead",
           characterUnitNumber = 1u,
           associatedCharactersUnitNumbers = emptyList(),
@@ -51,10 +45,13 @@ class FactorioEventTest : BehaviorSpec({
         ),
         eventType = "on_player_joined_game",
         modVersion = "0.0.4",
+        factorioVersion = "1.1.48",
         tick = 1278458u
       )
 
       actual shouldBeEqualToComparingFields expected
+
+      expected.data.objectName shouldBe "LuaPlayer"
     }
   }
 
@@ -70,6 +67,7 @@ class FactorioEventTest : BehaviorSpec({
         },
         "event_type": "on_tick",
         "mod_version": "0.0.4",
+        "factorio_version": "1.1.48",
         "tick": 1278480
       }
     """.trimIndent()
@@ -78,17 +76,19 @@ class FactorioEventTest : BehaviorSpec({
 
       val expected = FactorioEvent(
         data = SurfaceData(
-          objectName = "LuaSurface",
           name = "nauvis",
           index = 1u,
           daytime = 0.8390800000511436707029133685864508152008056640625
         ),
         eventType = "on_tick",
         modVersion = "0.0.4",
+        factorioVersion = "1.1.48",
         tick = 1278480u
       )
 
       actual shouldBeEqualToComparingFields expected
+
+      expected.data.objectName shouldBe "LuaSurface"
     }
   }
 
@@ -112,6 +112,7 @@ class FactorioEventTest : BehaviorSpec({
         },
         "event_type": "on_player_changed_position",
         "mod_version": "0.0.6",
+        "factorio_version": "1.1.48",
         "tick": 1384507
       }
     """.trimIndent()
@@ -120,7 +121,6 @@ class FactorioEventTest : BehaviorSpec({
 
       val expected = FactorioEvent(
         data = EntityData(
-          objectName = "LuaEntity",
           name = "character",
           type = "character",
           active = true,
@@ -136,10 +136,13 @@ class FactorioEventTest : BehaviorSpec({
         ),
         eventType = "on_player_changed_position",
         modVersion = "0.0.6",
+        factorioVersion = "1.1.48",
         tick = 1384507u
       )
 
       actual shouldBeEqualToComparingFields expected
+
+      expected.data.objectName shouldBe "LuaEntity"
     }
   }
 

modules/infra-factorio-client/build.gradle.kts

@@ -45,7 +45,7 @@ val clientLaunch by tasks.registering(Exec::class) {
   onlyIf(!isFactorioRunning())
 
   dependsOn(deployModToClient)
-  mustRunAfter(clientKill)
+  mustRunAfter(clientKill, ":modules:infra-factorio-server:processRun")
 
   commandLine = parseSpaceSeparatedArgs(
 //    """explorer "steam://rungameid/$factorioGameId// --mp-connect localhost/" """ // not working

modules/infra-factorio-server/build.gradle.kts

@@ -1,5 +1,6 @@
 import dev.adamko.kafkatorio.gradle.asConsumer
 import dev.adamko.kafkatorio.gradle.factorioModAttributes
+import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
 
 plugins {
   id("dev.adamko.kafkatorio.infra.docker-compose")
@@ -29,6 +30,10 @@ val deployModToServer by tasks.registering(Copy::class) {
   }
 }
 
+tasks.dockerDown {
+  commandLine = parseSpaceSeparatedArgs(""" docker-compose stop """)
+}
+
 tasks.dockerUp {
   dependsOn(
     deployModToServer,

modules/infra-kafka-cluster/src/docker-compose.yml

@@ -8,6 +8,7 @@ services:
     restart: "unless-stopped"
     build:
       dockerfile: "kafka.Dockerfile"
+      context: "."
       target: "kafka-kraft"
       args:
         KAFKA_DL_URL: "https://dlcdn.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz"

modules/infra-kafka-pipe/src/docker-compose.yml

@@ -6,6 +6,7 @@ services:
     container_name: "kafka-pipe"
     build:
       dockerfile: "kafka-pipe.Dockerfile"
+      context: "."
     restart: "unless-stopped"
     privileged: true
     networks: