packet size improvements

- change structure resource packets
- make more booleans default to false
- use encode_string - massive decrease in packet size!!

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/entities.kt

@@ -18,41 +18,23 @@ sealed interface FactorioEntityData {
 
   /** Non-specific entity data */
   @Serializable
-  @SerialName("kafkatorio.entity.FactorioEntityData")
+  @SerialName("kafkatorio.entity.FactorioEntityDataStandard")
   data class Standard(
     override val protoId: PrototypeId,
-    val status: Status? = null,
-
-    val unitNumber: UnitNumber? = null,
-
     override val position: MapEntityPosition,
+
     val graphicsVariation: UByte? = null,
     val health: Float? = null,
-    val isActive: Boolean? = null,
+    val isActive: Boolean = false,
-    val isRotatable: Boolean? = null,
+    val isRotatable: Boolean = false,
     val lastUser: UInt? = null,
-  ) : FactorioEntityData {
+    val status: Status? = null,
-    constructor(
+    val unitNumber: UnitNumber? = null,
-      position: MapEntityPosition,
+  ) : FactorioEntityData
-      protoId: PrototypeId,
-      element: FactorioEntityUpdateElement.Entity
-    ) : this(
-      protoId = protoId,
-      position = position,
-
-      status = element.status,
-      unitNumber = element.unitNumber,
-      graphicsVariation = element.graphicsVariation,
-      health = element.health,
-      isActive = element.isActive,
-      isRotatable = element.isRotatable,
-      lastUser = element.lastUser
-    )
-  }
 
 
   @Serializable
-  @SerialName("kafkatorio.entity.FactorioResourceData")
+  @SerialName("kafkatorio.entity.FactorioEntityDataResource")
   data class Resource(
     override val protoId: PrototypeId,
 
@@ -60,19 +42,7 @@ sealed interface FactorioEntityData {
     val amount: UInt,
     /** Count of initial resource units contained. `null` if resource is not infinite. */
     val initialAmount: UInt? = null,
-  ) : FactorioEntityData {
+  ) : FactorioEntityData
-    constructor(
-      position: MapEntityPosition,
-      protoId: PrototypeId,
-      element: FactorioEntityUpdateElement.Resource
-    ) : this(
-      protoId = protoId,
-      position = position,
-
-      amount = element.amount,
-      initialAmount = element.initialAmount,
-    )
-  }
 
 
   /**
@@ -193,7 +163,7 @@ sealed interface FactorioEntityData {
 }
 
 
-sealed interface FactorioEntityUpdateDictionary<T : FactorioEntityUpdateElement>
+sealed interface FactorioEntityUpdateDictionary<T : Any>
   : Iterable<Pair<MapEntityPosition, T>> {
 
   val entitiesXY: Map<String, Map<String, T>>
@@ -203,28 +173,37 @@ sealed interface FactorioEntityUpdateDictionary<T : FactorioEntityUpdateElement>
 
   override fun iterator(): Iterator<Pair<MapEntityPosition, T>> =
     iterator {
-      entitiesXY.forEach { (x, column) ->
+      val invalid = mutableListOf<String>()
-        column.forEach { (y, entity) ->
+      entitiesXY.forEach { (xString, column) ->
-          yield(
+        val x = xString.toDoubleOrNull()
-            MapEntityPosition(x.toDouble(), y.toDouble()) to entity
+        column.forEach { (yString, entity) ->
-          )
+          val y = yString.toDoubleOrNull()
+          if (x == null || y == null) {
+            invalid += "$xString/$yString"
+          } else {
+            yield(MapEntityPosition(x, y) to entity)
+          }
         }
       }
+
+      if (invalid.isNotEmpty()) {
+        println("WARNING FactorioEntityUpdateDictionary contained invalid x/y coords: ${invalid.joinToString()}")
+      }
     }
 
   @Serializable
-  @SerialName("kafkatorio.entity.FactorioEntityUpdateResourceDictionary")
+  @SerialName("kafkatorio.entity.FactorioEntityUpdateResourceAmountDictionary")
   @JvmInline
-  value class Resource(
+  value class ResourceAmounts(
-    override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Resource>>
+    override val entitiesXY: Map<String, Map<String, UInt>> = emptyMap()
-  ) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Resource>
+  ) : FactorioEntityUpdateDictionary<UInt>
 
   @Serializable
   @SerialName("kafkatorio.entity.FactorioEntityUpdateEntityDictionary")
   @JvmInline
   value class Entity(
-    override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Entity>>
+    override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Standard>>
-  ) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Entity>
+  ) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Standard>
 
 }
 
@@ -232,24 +211,16 @@ sealed interface FactorioEntityUpdateDictionary<T : FactorioEntityUpdateElement>
 sealed interface FactorioEntityUpdateElement {
 
   @Serializable
-  @SerialName("kafkatorio.entity.FactorioEntityUpdateResource")
+  @SerialName("kafkatorio.entity.FactorioEntityUpdateElementStandard")
-  data class Resource(
+  data class Standard(
-    val amount: UInt,
-    /** Count of initial resource units contained. `null` if resource is not infinite. */
-    val initialAmount: UInt? = null,
-  ) : FactorioEntityUpdateElement
-
-  @Serializable
-  @SerialName("kafkatorio.entity.FactorioEntityUpdateEntity")
-  data class Entity(
     val status: FactorioEntityData.Status? = null,
 
     val unitNumber: UnitNumber? = null,
 
     val graphicsVariation: UByte? = null,
     val health: Float? = null,
-    val isActive: Boolean? = null,
+    val isActive: Boolean = false,
-    val isRotatable: Boolean? = null,
+    val isRotatable: Boolean = false,
     val lastUser: UInt? = null,
   ) : FactorioEntityUpdateElement
 

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioPrototypes.kt

@@ -24,7 +24,7 @@ sealed interface FactorioPrototype {
     val collisionMasks: LuaJsonList<String>,
     val order: String,
     /** Can the tile be mined for resources? */
-    val canBeMined: Boolean,
+    val canBeMined: Boolean = false,
   ) : FactorioPrototype
 
 
@@ -63,7 +63,7 @@ sealed interface FactorioPrototype {
     @Serializable
     @SerialName("kafkatorio.prototype.EntityMiningProperties")
     data class MiningProperties(
-      val canBeMined: Boolean,
+      val canBeMined: Boolean = false,
 //      @Contextual
       val products: LuaJsonList<MinedProduct>?,
     )

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketInstant.kt

@@ -23,8 +23,8 @@ data class ConfigurationUpdate(
   val factorioData: ConfigurationUpdateGameData,
 //  @Contextual
   val allMods: LuaJsonList<ConfigurationUpdateModData>,
-  val modStartupSettingsChange: Boolean,
+  val modStartupSettingsChange: Boolean = false,
-  val migrationApplied: Boolean,
+  val migrationApplied: Boolean = false,
 ) : KafkatorioInstantPacketData() {
 
   @Serializable

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketKeyed.kt

@@ -7,6 +7,7 @@ import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
 import dev.adamko.kafkatorio.schema.common.EventName
 import dev.adamko.kafkatorio.schema.common.FactorioEntityData
 import dev.adamko.kafkatorio.schema.common.FactorioEntityUpdateDictionary
+import dev.adamko.kafkatorio.schema.common.FactorioEntityUpdateDictionary.ResourceAmounts
 import dev.adamko.kafkatorio.schema.common.ForceIndex
 import dev.adamko.kafkatorio.schema.common.MapChunkPosition
 import dev.adamko.kafkatorio.schema.common.MapEntityPosition
@@ -135,7 +136,7 @@ data class PlayerUpdate(
   val kickedReason: String? = null,
   val disconnectReason: String? = null,
   /** `true` when a player is removed (deleted) from the game */
-  val isRemoved: Boolean? = null,
+  val isRemoved: Boolean = false,
 ) : KafkatorioKeyedPacketData()
 
 
@@ -166,7 +167,14 @@ data class MapChunkEntityUpdate(
       FactorioEntityData.Standard(
         protoId = key.protoId,
         position = position,
-        element = entity,
+
+        status = entity.status,
+        unitNumber = entity.unitNumber,
+        graphicsVariation = entity.graphicsVariation,
+        health = entity.health,
+        isActive = entity.isActive,
+        isRotatable = entity.isRotatable,
+        lastUser = entity.lastUser
       )
     }
 }
@@ -191,15 +199,20 @@ data class MapChunkResourceUpdate(
 
   override val events: Map<EventName, LuaJsonList<Tick>>? = null,
 
-  val resourcesXY: FactorioEntityUpdateDictionary.Resource,
+  // Optimise JSON size by storing amounts separately.
+  val amounts: ResourceAmounts = ResourceAmounts(),
+  // Initial amount is only available for infinite resources, and infinite resources (oil) are more
+  // uncommon, so there are unlikely to be many per chunk
+  val initialAmounts: ResourceAmounts = ResourceAmounts(),
 ) : KafkatorioKeyedPacketData() {
 
   fun resources(): List<FactorioEntityData.Resource> =
-    resourcesXY.map { (position, resource) ->
+    amounts.map { (position, amount) ->
       FactorioEntityData.Resource(
         protoId = key.protoId,
         position = position,
-        element = resource,
+        amount = amount,
+        initialAmount = initialAmounts[position],
       )
     }
 }

modules/events-library/src/jvmMain/kotlin/dev/adamko/kafkatorio/schema/kt2ts2.kt

@@ -99,7 +99,7 @@ class MySourceCodeGenerator(
       (element.typeRef as? TsTypeRef.Literal)?.element is TsLiteral.Custom
       || element.id.name == "Tick"
       || element.id.name == "FactorioEntityUpdateEntityDictionary"
-      || element.id.name == "FactorioEntityUpdateResourceDictionary"
+      || element.id.name == "FactorioEntityUpdateResourceAmountDictionary"
     ) {
       val aliasedRef = generateTypeReference(element.typeRef)
       """

modules/events-mod/src/generated/lua/main/emitting/PacketEmitter.lua

@@ -15,7 +15,8 @@ function KafkatorioPacketEmitter.prototype.emitKeyedPacket(self, data)
 end
 function KafkatorioPacketEmitter.emitPacket(self, packet)
     local data = game.table_to_json(packet)
-    localised_print("KafkatorioPacket: " .. data)
+    local encoded = game.encode_string(data)
+    print("KafkatorioPacket encoded::: " .. tostring(encoded))
 end
 local PacketEmitter = __TS__New(____exports.KafkatorioPacketEmitter)
 ____exports.default = PacketEmitter

modules/events-mod/src/generated/lua/main/events/converters.lua

@@ -42,13 +42,6 @@ do
         end
         return masks
     end
-    function Converters.convertResourceEntity(entity)
-        if entity.type ~= "resource" then
-            return nil
-        else
-            return {amount = entity.amount, initialAmount = entity.initial_amount}
-        end
-    end
     local function convertEntityStatus(status)
         if status == nil then
             return nil

modules/events-mod/src/generated/lua/main/events/handlers/EntityUpdateHandler.lua

@@ -142,16 +142,23 @@ function EntityUpdatesHandler.throttleResourcesUpdate(self, eventName, eventTick
                 local ____data_events_eventName_17 = data.events[eventName]
                 ____data_events_eventName_17[#____data_events_eventName_17 + 1] = eventTick
                 for ____, entity in ipairs(resourceEntities) do
-                    local resourceUpdate = Converters.convertResourceEntity(entity)
+                    if data.amounts == nil then
-                    if resourceUpdate ~= nil then
+                        data.amounts = {}
-                        if data.resourcesXY == nil then
+                    end
-                            data.resourcesXY = {}
+                    local ____data_amounts_18, ____tostring_result_19 = data.amounts, tostring(entity.position.x)
+                    if ____data_amounts_18[____tostring_result_19] == nil then
+                        ____data_amounts_18[____tostring_result_19] = {}
+                    end
+                    data.amounts[tostring(entity.position.x)][tostring(entity.position.y)] = entity.amount
+                    if entity.initial_amount ~= nil then
+                        if data.initialAmounts == nil then
+                            data.initialAmounts = {}
                         end
-                        local ____data_resourcesXY_18, ____tostring_result_19 = data.resourcesXY, tostring(entity.position.x)
+                        local ____data_initialAmounts_20, ____tostring_result_21 = data.initialAmounts, tostring(entity.position.x)
-                        if ____data_resourcesXY_18[____tostring_result_19] == nil then
+                        if ____data_initialAmounts_20[____tostring_result_21] == nil then
-                            ____data_resourcesXY_18[____tostring_result_19] = {}
+                            ____data_initialAmounts_20[____tostring_result_21] = {}
                         end
-                        data.resourcesXY[tostring(entity.position.x)][tostring(entity.position.y)] = resourceUpdate
+                        data.initialAmounts[tostring(entity.position.x)][tostring(entity.position.y)] = entity.initial_amount
                     end
                 end
             end

modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts

@@ -40,8 +40,8 @@ export namespace KafkatorioPacketData {
     type: KafkatorioPacketData.Type.ConfigurationUpdate;
     factorioData: ConfigurationUpdateGameData;
     allMods: ConfigurationUpdateModData[];
-    modStartupSettingsChange: boolean;
+    modStartupSettingsChange?: boolean;
-    migrationApplied: boolean;
+    migrationApplied?: boolean;
   }
   
   export interface ConsoleChatUpdate {
@@ -94,7 +94,8 @@ export namespace KafkatorioPacketData {
     type: KafkatorioPacketData.Type.MapChunkResourceUpdate;
     key: MapChunkResourceUpdateKey;
     events?: { [key: EventName]: Tick[] } | null;
-    resourcesXY: FactorioEntityUpdateResourceDictionary;
+    amounts?: FactorioEntityUpdateResourceAmountDictionary;
+    initialAmounts?: FactorioEntityUpdateResourceAmountDictionary;
   }
   
   export interface MapChunkTileUpdate {
@@ -132,7 +133,7 @@ export namespace KafkatorioPacketData {
     bannedReason?: string | null;
     kickedReason?: string | null;
     disconnectReason?: string | null;
-    isRemoved?: boolean | null;
+    isRemoved?: boolean;
   }
   
   export interface Error {
@@ -171,7 +172,7 @@ export interface MapChunkEntityUpdateKey {
   chunkPosition: MapChunkPosition;
 }
 
-export type FactorioEntityUpdateEntityDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateEntity } };
+export type FactorioEntityUpdateEntityDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateElementStandard } };
 
 export interface MapChunkResourceUpdateKey {
   protoId: PrototypeId;
@@ -179,7 +180,7 @@ export interface MapChunkResourceUpdateKey {
   chunkPosition: MapChunkPosition;
 }
 
-export type FactorioEntityUpdateResourceDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateResource } };
+export type FactorioEntityUpdateResourceAmountDictionary = { [key: string]: { [key: string]: UInt } };
 
 export interface MapChunkTileUpdateKey {
   chunkPosition: MapChunkPosition;
@@ -251,7 +252,7 @@ export namespace FactorioPrototype {
     layer: UInt;
     collisionMasks: string[];
     order: string;
-    canBeMined: boolean;
+    canBeMined?: boolean;
   }
 }
 
@@ -274,7 +275,7 @@ export interface EntityItemGroup {
 }
 
 export interface EntityMiningProperties {
-  canBeMined: boolean;
+  canBeMined?: boolean;
   products: MinedProduct[] | null;
 }
 
@@ -293,21 +294,16 @@ export enum ChunkSize {
 
 export type PrototypeKey = Int & { __PrototypeKey__: void };
 
-export interface FactorioEntityUpdateEntity {
+export interface FactorioEntityUpdateElementStandard {
   status?: FactorioEntityStatus | null;
   unitNumber?: UnitNumber | null;
   graphicsVariation?: UByte | null;
   health?: Float | null;
-  isActive?: boolean | null;
+  isActive?: boolean;
-  isRotatable?: boolean | null;
+  isRotatable?: boolean;
   lastUser?: UInt | null;
 }
 
-export interface FactorioEntityUpdateResource {
-  amount: UInt;
-  initialAmount?: UInt | null;
-}
-
 export type MinedProduct =
   | MinedProduct.MinedProductFluid
   | MinedProduct.MinedProductItem;

modules/events-mod/src/main/typescript/main/emitting/PacketEmitter.ts

@@ -26,8 +26,10 @@ export class KafkatorioPacketEmitter {
 
   /** Emit a serialised KafkatorioPacket */
   private static emitPacket<T extends KafkatorioPacket>(packet: T) {
-    let data = game.table_to_json(packet)
+    const data = game.table_to_json(packet)
-    localised_print(`KafkatorioPacket: ${data}`)
+    // print(`KafkatorioPacket::: ${data}`)
+    const encoded = game.encode_string(data)
+    print(`KafkatorioPacket encoded::: ${encoded}`)
     // rcon.print(`KafkatorioPacket: ${data}`)
   }
 

modules/events-mod/src/main/typescript/main/events/converters.ts

@@ -5,7 +5,6 @@ import {
   EntityMiningProperties,
   EventName,
   FactorioEntityStatus,
-  FactorioEntityUpdateResource,
   KafkatorioPacketData,
   MapBoundingBox,
   MapChunkPosition,
@@ -76,20 +75,6 @@ export namespace Converters {
   }
 
 
-  export function convertResourceEntity(
-      entity: LuaEntity
-  ): FactorioEntityUpdateResource | null {
-    if (entity.type != "resource") {
-      return null
-    } else {
-      return {
-        amount: entity.amount,
-        initialAmount: entity.initial_amount,
-      }
-    }
-  }
-
-
   function convertEntityStatus(
       status: defines.entity_status | undefined
   ): FactorioEntityStatus | null {

modules/events-mod/src/main/typescript/main/events/handlers/EntityUpdateHandler.ts

@@ -1,6 +1,6 @@
 import {
   EventName,
-  FactorioEntityUpdateEntity,
+  FactorioEntityUpdateElementStandard,
   KafkatorioPacketData,
   MapChunkPosition,
   PrototypeId,
@@ -94,7 +94,7 @@ export class EntityUpdatesHandler {
       chunkPosition: Converters.mapPositionToChunkPosition(entity.position)
     }
 
-    const entityUpdate: FactorioEntityUpdateEntity = {
+    const entityUpdate: FactorioEntityUpdateElementStandard = {
       unitNumber: entity.unit_number,
       graphicsVariation: entity.graphics_variation,
       health: entity.health,
@@ -140,7 +140,7 @@ export class EntityUpdatesHandler {
 
     for (const [protoId, resourceEntities] of pairs(entitiesByProtoId)) {
 
-      // create an update for each prototype ID
+      // create a MapChunkResourceUpdate for each prototype ID
 
       const entityUpdateKey: KafkatorioPacketData.MapChunkResourceUpdate["key"] = {
         protoId: protoId,
@@ -158,18 +158,20 @@ export class EntityUpdatesHandler {
             data.events[eventName].push(eventTick)
 
             for (const entity of resourceEntities) {
-              const resourceUpdate = Converters.convertResourceEntity(entity)
 
-              if (resourceUpdate != null) {
+              data.amounts ??= {}
-                data.resourcesXY ??= {}
+              data.amounts[`${entity.position.x}`] ??= {}
-                data.resourcesXY[`${entity.position.x}`] ??= {}
+              data.amounts[`${entity.position.x}`][`${entity.position.y}`] = entity.amount
-                data.resourcesXY[`${entity.position.x}`][`${entity.position.y}`] = resourceUpdate
+
+              if (entity.initial_amount != null) {
+                data.initialAmounts ??= {}
+                data.initialAmounts[`${entity.position.x}`] ??= {}
+                data.initialAmounts[`${entity.position.x}`][`${entity.position.y}`] = entity.initial_amount
               }
             }
           }
       )
     }
-
   }
 
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/KafkatorioTopology.kt

@@ -167,6 +167,7 @@ internal class KafkatorioTopology(
   }
 
 
+  /** Receive log messages from the syslog socket, and send them to a Kafka topic */
   private fun syslogProducer(
     id: String = "syslog-producer"
   ) {
@@ -187,10 +188,14 @@ internal class KafkatorioTopology(
     }
 
     syslogServer.messages
-      .filter { it.message?.startsWith("KafkatorioPacket:") == true }
+      .filter {
+        (it.message ?: "").run {
+          startsWith("KafkatorioPacket:::") || startsWith("KafkatorioPacket encoded:::")
+        }
+      }
       .onEach { syslogMsg ->
 
-        val kafkatorioMessage = syslogMsg.message?.substringAfter("KafkatorioPacket:")
+        val kafkatorioMessage = syslogMsg.message?.substringAfter(":::")
 
         val record = ProducerRecord<String, String>(
           TOPIC_SRC_SERVER_LOG,

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/factorioServerPackets.kt

@@ -14,7 +14,13 @@ import dev.adamko.kotka.extensions.producedAs
 import dev.adamko.kotka.extensions.streams.map
 import dev.adamko.kotka.extensions.streams.to
 import dev.adamko.kotka.kxs.serde
+import java.util.zip.Inflater
 import kotlinx.serialization.decodeFromString
+import okio.Buffer
+import okio.ByteString
+import okio.ByteString.Companion.decodeBase64
+import okio.buffer
+import okio.inflate
 import org.apache.kafka.common.serialization.Serdes
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.kstream.KStream
@@ -33,7 +39,8 @@ fun factorioServerPacketStream(
     )
   ).map("decode-packets") { serverId: String, value: String ->
     val packet = runCatching {
-      jsonMapper.decodeFromString<KafkatorioPacket>(value)
+      val decoded = decodeFactorioEncodedString(value)
+      jsonMapper.decodeFromString<KafkatorioPacket>(decoded)
     }.getOrElse { e ->
       KafkatorioPacket(
         modVersion = "unknown",
@@ -69,3 +76,18 @@ fun splitFactorioServerPacketStream(
       value.data.topicName
     }
 }
+
+
+/**
+ * Factorio `game.encode_string` zlib encodes a string, then base64's it. Here we do the reverse.
+ */
+private fun decodeFactorioEncodedString(
+  source: String
+): String {
+  val bytes: ByteString = source.decodeBase64() ?: error("failed to base64 decode $source")
+
+  return Buffer().write(bytes)
+    .inflate(Inflater())
+    .buffer()
+    .readString(Charsets.UTF_8)
+}