- wicked sick size optimisation of map-tiles update packet

- fix ts-generator recursive value class unwrapping
- move kafka state dir
- try fixing kafka streams for tile generation (simplifying it, and fixing some nullability problems)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt

@@ -1,7 +1,6 @@
 package dev.adamko.kafkatorio.events.schema
 
 import kotlin.jvm.JvmInline
-import kotlinx.serialization.EncodeDefault
 import kotlinx.serialization.Serializable
 
 
@@ -101,3 +100,41 @@ interface EntityIdentifiers {
   val name: String
   val type: String
 }
+
+
+/**
+ * Size-optimised 2D map of x/y tile positions and prototype-name.
+ *
+ * Each X/Y coordinate maps to an arbitrary [PrototypeKey] (which is only valid for any specific
+ * instance). [protos] is specific to each instance of [MapTileDictionary], can be used to convert from a
+ */
+@Serializable
+data class MapTileDictionary(
+  /** Map an X,Y coordinate a prototype name */
+  val tilesXY: Map<String, Map<String, PrototypeKey>>,
+  val protos: Map<PrototypeKey, PrototypeName>
+) {
+  fun toMapTileList(): List<MapTile> = buildList {
+    tilesXY.forEach { (xString, row) ->
+      row.forEach { (yString, protoIndex) ->
+        val x = xString.toIntOrNull()
+        val y = yString.toIntOrNull()
+        val protoName = protos[protoIndex]
+        if (x != null && y != null && protoName != null) {
+          add(MapTile(x, y, protoName))
+        }
+      }
+    }
+  }
+
+  companion object {
+    fun MapTileDictionary?.isNullOrEmpty(): Boolean {
+      return this == null || (tilesXY.isEmpty() && tilesXY.all { it.value.isEmpty() })
+    }
+  }
+}
+
+
+@Serializable
+@JvmInline
+value class PrototypeKey(val index: String)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioEventUpdates.kt

@@ -147,7 +147,7 @@ data class EntityUpdate(
   override val eventCounts: Map<String, UInt>? = null,
 
   val chunkPosition: MapChunkPosition? = null,
-  val graphicsVariation: UShort? = null,
+  val graphicsVariation: UByte? = null,
   val health: Float? = null,
   val isActive: Boolean? = null,
   val isRotatable: Boolean? = null,
@@ -180,11 +180,8 @@ data class MapChunkUpdate(
   val player: PlayerIndex? = null,
   val robot: EntityIdentifiers? = null,
   val force: ForceIndex? = null,
-  /** updated tiles - might be partial */
-  // note: this must be a List, not a Map<Location, ProtoName>
-  // because JSON can't have non-string keys.
-  @Contextual
-  val tiles: List<MapTile>? = null,
+  /** updated tiles - might be partial and not all tiles in the chunk */
+  val tileDictionary: MapTileDictionary? = null,
   val isDeleted: Boolean? = null,
 ) : FactorioEventUpdate(), MapChunkUpdateKey {
   @EncodeDefault

modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/ValueClassTransformer.kt

@@ -1,11 +1,15 @@
 package dev.adamko.kafkatorio.events.schema
 
+import kotlin.contracts.ExperimentalContracts
+import kotlin.contracts.contract
 import kotlin.reflect.KClass
+import kotlin.reflect.KClassifier
 import kotlin.reflect.KProperty
 import kotlin.reflect.KType
 import kotlin.reflect.KTypeProjection
 import kotlin.reflect.full.createType
 import kotlin.reflect.full.declaredMemberProperties
+import kotlin.reflect.full.starProjectedType
 import kotlin.reflect.full.withNullability
 import me.ntrrgc.tsGenerator.ClassTransformer
 
@@ -14,54 +18,77 @@ import me.ntrrgc.tsGenerator.ClassTransformer
  */
 object ValueClassTransformer : ClassTransformer {
 
+  private val mappedValueClasses: MutableMap<KType, KType> = mutableMapOf(
+    // don't unwrap unsigned numbers - they're mapped to Factorio types
+    UInt::class.starProjectedType to UInt::class.starProjectedType,
+    UByte::class.starProjectedType to UByte::class.starProjectedType,
+    UShort::class.starProjectedType to UShort::class.starProjectedType,
+    ULong::class.starProjectedType to ULong::class.starProjectedType,
+  )
+
   override fun transformPropertyType(type: KType, property: KProperty<*>, klass: KClass<*>): KType {
-
-    val classifier = type.classifier ?: return super.transformPropertyType(type, property, klass)
-
-    val valueClassType = unwrapValueClassType(type)
-
-    if (valueClassType != null) {
-      println("$klass has a value class $type -> $valueClassType")
-      return valueClassType
+    val unwrappedType =
+      unwrapValueClassType(type, property, klass)
+        ?: unwrapArgs(type, property, klass)
+    return when (unwrappedType) {
+      null -> super.transformPropertyType(type, property, klass)
+      else -> super.transformPropertyType(unwrappedType, property, klass)
     }
+  }
 
-    val hasValueClassTypeParameters = type.arguments
-      .mapNotNull { it.type?.classifier }
-      .filterIsInstance<KClass<*>>()
-      .any { it.isValue }
+  private fun unwrapArgs(type: KType, property: KProperty<*>, klass: KClass<*>): KType? {
 
-    if (hasValueClassTypeParameters) {
+    val classifier: KClassifier = type.classifier ?: return type
 
-      val args = type.arguments.map { arg ->
-        when (val vct = unwrapValueClassType(arg.type)) {
-          null -> arg
-          else -> KTypeProjection(arg.variance, vct.withNullability(arg.type!!.isMarkedNullable))
-        }
+    return if (type.arguments.isNotEmpty()) {
+      println("unwrapping args for $type - ${type.arguments}")
+
+      val args = type.arguments.map { arg: KTypeProjection ->
+        // recursively map each argument
+        val argType = arg.type!!
+        val converted = transformPropertyType(argType, property, klass)
+        KTypeProjection(arg.variance, converted.withNullability(argType.isMarkedNullable))
       }
 
-      println("$klass has a prop with value class type parameters ${type.arguments} -> $args")
-
-      return classifier.createType(
+      classifier.createType(
         arguments = args,
         nullable = type.isMarkedNullable,
         annotations = type.annotations,
       )
-    }
-
-    return super.transformPropertyType(type, property, klass)
-  }
-
-  private fun unwrapValueClassType(type: KType?): KType? {
-    val klass = type?.classifier as? KClass<*> ?: return null
-
-    return when {
-      klass.isValue -> klass
-        .declaredMemberProperties
-        .first()
-        .returnType
-        .withNullability(type.isMarkedNullable)
-
-      else          -> null
+    } else {
+      null
     }
   }
+
+  private fun unwrapValueClassType(type: KType, property: KProperty<*>, klass: KClass<*>): KType? {
+
+    val valueKlass: KClass<*> = type.classifier as? KClass<*> ?: return null
+
+    return if (isValueClass(type)) {
+      val mappedType = mappedValueClasses.getOrPut(type.withNullability(false)) {
+
+        val result = valueKlass
+          .declaredMemberProperties
+          .first()
+          .returnType
+          .withNullability(type.isMarkedNullable)
+
+        println("unwrapping value class $type -> $result")
+
+        // recursively unwrap...
+        val transformed = transformPropertyType(result, property, klass)
+        transformed.withNullability(false)
+      }
+      mappedType.withNullability(type.isMarkedNullable)
+    } else {
+      null
+    }
+  }
+
+  @OptIn(ExperimentalContracts::class)
+  private fun isValueClass(type: KType?): Boolean {
+    contract { returnsNotNull() implies (type != null) }
+    return (type?.classifier as? KClass<*>)?.isValue == true
+  }
+
 }

modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts.kt

@@ -21,12 +21,9 @@ fun main(args: Array<String>) {
         add(FactorioPrototypes::class)
         add(FactorioConfigurationUpdate::class)
 
-
         add(FactorioEventUpdatePacket::class)
         addAll(FactorioEventUpdate::class.sealedSubclasses)
         addAll(FactorioEventUpdateKey::class.sealedSubclasses)
-//        addAll(FactorioEventUpdateData::class.sealedSubclasses)
-
       },
       mappings = mapOf(
         // builtin Factorio numeric types > `typed-factorio/generated/builtin-types.d.ts`
@@ -74,7 +71,6 @@ private fun splitDefinitions(definitions: String): Map<String, String> {
     .split("\n\n")
     .groupBy { def ->
 
-
       when {
 
         "interface FactorioObjectData " in def ||