Render buildings and resources

- create LuaJsonList to handle empty Lua JSON lists being sent as empty objects
- fix exporting of resources and entities
- split up map tiles into different layer directories
- add 'chunk size' to MapChunkPosition
- move tests to correct packages
- bump tstl
- decrease tstl logging
- remove some defined, but unused, kafka topics
- syslog, rest, websocket servers & web-map: decrease logging
- webmap: add additional layers for buildings and resources

build.gradle.kts

@@ -47,8 +47,8 @@ val startInfra by tasks.registering {
   group = project.name
 
   dependsOn(
-    ":modules:infra-kafka-pipe:processRun",
-    ":modules:infra-kafka-cluster:processRun",
+//    ":modules:infra-kafka-pipe:processRun",
+    runKafkaCluster
 //    ":modules:infra-kafka-connect:processRun",
   )
 }
@@ -74,3 +74,30 @@ val runWebMap by tasks.registering {
 
   dependsOn(":modules:web-map:browserDevelopmentRun")
 }
+
+
+val runKafkaCluster by tasks.registering {
+  group = project.name
+
+  dependsOn(
+    ":modules:infra-kafka-cluster:processRun",
+  )
+}
+
+
+val runFactorioServer by tasks.registering {
+  group = project.name
+
+  dependsOn(
+    ":modules:infra-factorio-server:processRun",
+  )
+}
+
+
+val runFactorioClient by tasks.registering {
+  group = project.name
+
+  dependsOn(
+    ":modules:infra-factorio-client:processRun",
+  )
+}

gradle.properties

@@ -10,7 +10,7 @@ org.gradle.unsafe.configuration-cache-problems=warn
 
 org.gradle.kotlin.dsl.precompiled.accessors.strict=true
 
-org.gradle.parallel=true
+#org.gradle.parallel=true # helps with pRoJeCtS mUsT bE cOnFiGuRiNg
 
 # pRoJeCtS mUsT bE cOnFiGuRiNg
 org.gradle.configureondemand=false

modules/events-library/build.gradle.kts

@@ -129,7 +129,7 @@ val generateTypeScript by tasks.registering(GenerateTypeScriptTask::class) {
 //    kotlinMainDependencies,
   )
   output.set(layout.buildDirectory.dir("generated/typescript"))
-  mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2ts2Kt")
+  mainClass.set("dev.adamko.kafkatorio.schema.Kt2ts2Kt")
   args(temporaryDir.canonicalPath)
 }
 

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/library/json.kt

@@ -1,19 +1,17 @@
 package dev.adamko.kafkatorio.library
 
-import kotlin.reflect.KClass
-import kotlinx.serialization.DeserializationStrategy
+import kotlin.jvm.JvmInline
 import kotlinx.serialization.KSerializer
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.SerializationException
 import kotlinx.serialization.builtins.ListSerializer
+import kotlinx.serialization.descriptors.SerialDescriptor
+import kotlinx.serialization.encoding.Decoder
+import kotlinx.serialization.encoding.Encoder
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonArray
-import kotlinx.serialization.json.JsonContentPolymorphicSerializer
-import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonDecoder
 import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.JsonTransformingSerializer
-import kotlinx.serialization.json.buildJsonObject
-import kotlinx.serialization.json.contentOrNull
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
 import kotlinx.serialization.modules.SerializersModule
 
 
@@ -22,55 +20,60 @@ val jsonMapperKafkatorio = Json {
   prettyPrintIndent = "  "
 
   serializersModule = SerializersModule {
-    contextual(List::class) { args -> FactorioJsonListSerializer(args[0]) }
+//    contextual(List::class) { args -> FactorioListJsonSerializer(args[0]) }
   }
 }
 
 
+/** @see LuaJsonListSerializer */
+@Serializable(with = LuaJsonListSerializer::class)
+@JvmInline
+value class LuaJsonList<T>(private val list: List<T>) : List<T> by list
+
+
 /**
- * Factorio outputs an *empty* list as an *empty* Json object. This serde performs the same
- * conversion.
+ * Factorio outputs **empty** lists `[]` as *empty* Json objects `{}`.
+ *
+ * When deserializing JSON, this serializer transforms any incoming empty objects to empty
+ * [array][JsonArray]s.
  */
-class FactorioJsonListSerializer<T>(dataSerializer: KSerializer<T>) :
-  JsonTransformingSerializer<List<T>>(ListSerializer(dataSerializer)) {
+class LuaJsonListSerializer<T>(
+  elementSerializer: KSerializer<T>
+) : KSerializer<LuaJsonList<T>> {
 
-  override fun transformDeserialize(element: JsonElement): JsonElement {
-    return when (element) {
-      is JsonObject -> JsonArray(emptyList())
-      else          -> element
+  private val listSerializer: KSerializer<List<T>> = ListSerializer(elementSerializer)
+
+  override val descriptor: SerialDescriptor =
+    SerialDescriptor("kafkatorio.LuaList", listSerializer.descriptor)
+
+  override fun deserialize(decoder: Decoder): LuaJsonList<T> = LuaJsonList(
+    when (decoder) {
+      is JsonDecoder -> deserializeJson(decoder)
+      else           -> decoder.decodeSerializableValue(listSerializer)
     }
+  )
+
+  private fun deserializeJson(decoder: JsonDecoder): List<T> =
+    when (val element = decoder.decodeJsonElement()) {
+      is JsonObject -> when {
+        element.entries.isEmpty() -> emptyList()
+        else                      -> deserializeError(
+          "expected empty Json object, but contained ${element.entries.size} entries ${element.entries}"
+        )
+      }
+      is JsonArray  -> decoder.json.decodeFromJsonElement(listSerializer, element)
+      else          -> deserializeError(
+        "expected empty Json object or a Json array, but was $element"
+      )
     }
 
-  override fun transformSerialize(element: JsonElement): JsonElement {
-    return when {
-      element is JsonArray && element.isEmpty() -> buildJsonObject { }
-      else                                      -> element
-    }
+  override fun serialize(encoder: Encoder, value: LuaJsonList<T>) {
+    encoder.encodeSerializableValue(listSerializer, value)
   }
 
-}
-
-
-interface SerializerProvider<T : Any> {
-  val serializer: KSerializer<out T>
-}
-
-open class SerializerProviderDeserializer<T : Any>(
-  private val baseClass: KClass<T>,
-  private val key: String,
-  private val selector: (String) -> SerializerProvider<out T>?,
-) : JsonContentPolymorphicSerializer<T>(baseClass) {
-
-  override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out T> {
-
-    val type = element
-      .jsonObject[key]
-      ?.jsonPrimitive
-      ?.contentOrNull
-      ?.let { json -> selector(json) }
-
-    requireNotNull(type) { "Unknown ${baseClass.simpleName} $key: $element" }
-
-    return type.serializer
+  companion object {
+    private fun deserializeError(msg: String): Nothing = throw SerializationException(
+      "Error deserializing LuaList: $msg"
+    )
   }
 }

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/library/positionOperations.kt

@@ -0,0 +1,103 @@
+package dev.adamko.kafkatorio.library
+
+import dev.adamko.kafkatorio.schema.common.ChunkSize
+import dev.adamko.kafkatorio.schema.common.MapChunkPosition
+import dev.adamko.kafkatorio.schema.common.MapEntityPosition
+import dev.adamko.kafkatorio.schema.common.MapTilePosition
+import kotlin.math.floor
+
+
+operator fun MapTilePosition.times(factor: Int): MapTilePosition =
+  MapTilePosition(x * factor, y * factor)
+
+
+operator fun MapTilePosition.div(divisor: Int): MapTilePosition =
+  MapTilePosition(x / divisor, y / divisor)
+
+
+operator fun MapTilePosition.plus(addend: Int): MapTilePosition =
+  MapTilePosition(x + addend, y + addend)
+
+
+operator fun MapTilePosition.plus(addend: MapTilePosition): MapTilePosition =
+  MapTilePosition(x + addend.x, y + addend.y)
+
+
+operator fun MapTilePosition.minus(subtrahend: MapTilePosition): MapTilePosition =
+  MapTilePosition(x - subtrahend.x, y - subtrahend.y)
+
+
+/** Get the [MapChunkPosition] of the tile */
+fun MapTilePosition.toMapChunkPosition(chunkSize: ChunkSize): MapChunkPosition =
+  MapChunkPosition(
+    x = floor(x.toDouble() / chunkSize.lengthInTiles.toDouble()).toInt(),
+    y = floor(y.toDouble() / chunkSize.lengthInTiles.toDouble()).toInt(),
+    chunkSize = chunkSize,
+  )
+
+
+/** Get the [MapTilePosition] of the entity */
+fun MapEntityPosition.toMapTilePosition(): MapTilePosition =
+  MapTilePosition(
+    x = floor(x).toInt(),
+    y = floor(y).toInt(),
+  )
+
+
+/** Get the [MapChunkPosition] of the entity */
+fun MapEntityPosition.toMapChunkPosition(chunkSize: ChunkSize): MapChunkPosition =
+  MapChunkPosition(
+    x = floor(x / chunkSize.lengthInTiles.toDouble()).toInt(),
+    y = floor(y / chunkSize.lengthInTiles.toDouble()).toInt(),
+    chunkSize = chunkSize,
+  )
+
+
+fun MapChunkPosition.resize(chunkSize: ChunkSize): MapChunkPosition =
+  leftTopTile().toMapChunkPosition(chunkSize)
+
+
+operator fun MapChunkPosition.times(factor: Int): MapChunkPosition =
+  copy(x = x * factor, y = y * factor)
+
+
+operator fun MapChunkPosition.plus(addend: Int): MapChunkPosition =
+  copy(x = x + addend, y = y + addend)
+
+
+operator fun MapChunkPosition.minus(subtrahend: Int): MapChunkPosition =
+  copy(x = x - subtrahend, y = y - subtrahend)
+
+
+fun MapChunkPosition.leftTopTile(): MapTilePosition =
+  MapTilePosition(
+    x * chunkSize.lengthInTiles,
+    y * chunkSize.lengthInTiles,
+  )
+
+
+fun MapChunkPosition.rightBottomTile(): MapTilePosition =
+  leftTopTile() + (chunkSize.lengthInTiles - 1)
+
+
+/** Iterate over all possible [MapTilePosition]s in the chunk. */
+fun MapChunkPosition.iterateTiles(): Iterator<MapTilePosition> {
+  val leftTop = leftTopTile()
+  val rightBottom = rightBottomTile()
+
+  return iterator {
+    (leftTop.x..rightBottom.x).forEach { x ->
+      (leftTop.y..rightBottom.y).forEach { y ->
+        yield(MapTilePosition(x, y))
+      }
+    }
+  }
+}
+
+
+//val MapChunkPosition.rightBottomTile: MapTilePosition
+//  get() = leftTopTile + (MAP_CHUNK_SIZE - 1)
+
+//
+//val MapChunkPosition.boundingBox: MapBoundingBox
+//  get() = MapBoundingBox(leftTopTile, rightBottomTile)

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/colours.kt

@@ -52,6 +52,7 @@ data class ColourHex(
 ) {
   companion object {
     val TRANSPARENT = ColourHex(0u, 0u, 0u, 0u)
+    val WHITE = ColourHex(UByte.MAX_VALUE, UByte.MAX_VALUE, UByte.MAX_VALUE, UByte.MAX_VALUE)
   }
 }
 

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/entities.kt

@@ -1,24 +1,29 @@
 package dev.adamko.kafkatorio.schema.common
 
+import kotlin.jvm.JvmInline
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
 
+// TODO change all booleans to be optional, and default to 'false' (also in events-mod)
+
+
 @Serializable
 @SerialName("kafkatorio.entity.FactorioEntityData")
 sealed interface FactorioEntityData {
 
   val protoId: PrototypeId
-  val status: Status?
 
   val position: MapEntityPosition
 
   /** Non-specific entity data */
   @Serializable
-  @SerialName("kafkatorio.entity.FactorioEntityData.Standard")
+  @SerialName("kafkatorio.entity.FactorioEntityData")
   data class Standard(
     override val protoId: PrototypeId,
-    override val status: Status? = null,
+    val status: Status? = null,
+
+    val unitNumber: UnitNumber? = null,
 
     override val position: MapEntityPosition,
     val graphicsVariation: UByte? = null,
@@ -26,121 +31,230 @@ sealed interface FactorioEntityData {
     val isActive: Boolean? = null,
     val isRotatable: Boolean? = null,
     val lastUser: UInt? = null,
-    val localisedDescription: String? = null,
-    val localisedName: String? = null,
-  ) : FactorioEntityData
+  ) : FactorioEntityData {
+    constructor(
+      position: MapEntityPosition,
+      protoId: PrototypeId,
+      element: FactorioEntityUpdateElement.Entity
+    ) : this(
+      protoId = protoId,
+      position = position,
+
+      status = element.status,
+      unitNumber = element.unitNumber,
+      graphicsVariation = element.graphicsVariation,
+      health = element.health,
+      isActive = element.isActive,
+      isRotatable = element.isRotatable,
+      lastUser = element.lastUser
+    )
+  }
 
 
   @Serializable
-  @SerialName("kafkatorio.entity.FactorioEntityData.Resource")
+  @SerialName("kafkatorio.entity.FactorioResourceData")
   data class Resource(
     override val protoId: PrototypeId,
-    override val status: Status? = null,
 
     override val position: MapEntityPosition,
     val amount: UInt,
     /** Count of initial resource units contained. `null` if resource is not infinite. */
     val initialAmount: UInt? = null,
-  ) : FactorioEntityData
+  ) : FactorioEntityData {
+    constructor(
+      position: MapEntityPosition,
+      protoId: PrototypeId,
+      element: FactorioEntityUpdateElement.Resource
+    ) : this(
+      protoId = protoId,
+      position = position,
+
+      amount = element.amount,
+      initialAmount = element.initialAmount,
+    )
+  }
 
 
+  /**
+   * [`https://lua-api.factorio.com/latest/defines.html#defines.entity_status`](https://lua-api.factorio.com/latest/defines.html#defines.entity_status)
+   */
+  @Suppress("unused") // not directly used at the moment - maybe later
   @Serializable
-  @SerialName("kafkatorio.entity.FactorioEntityData.EntityStatus")
+  @SerialName("kafkatorio.entity.FactorioEntityData.FactorioEntityStatus")
   enum class Status {
-    CANT_DIVIDE_SEGMENTS,
-    CHARGING,
-    CLOSED_BY_CIRCUIT_NETWORK,
-    DISABLED,
-    DISABLED_BY_CONTROL_BEHAVIOR,
-    DISABLED_BY_SCRIPT,
-    DISCHARGING,
-    FLUID_INGREDIENT_SHORTAGE,
-    FULLY_CHARGED,
-    FULL_OUTPUT,
-    ITEM_INGREDIENT_SHORTAGE,
-    LAUNCHING_ROCKET,
-    LOW_INPUT_FLUID,
-    LOW_POWER,
-    LOW_TEMPERATURE,
-    MARKED_FOR_DECONSTRUCTION,
-    MISSING_REQUIRED_FLUID,
-    MISSING_SCIENCE_PACKS,
-    NETWORKS_CONNECTED,
-    NETWORKS_DISCONNECTED,
-    NORMAL,
-    NOT_CONNECTED_TO_RAIL,
-    NOT_PLUGGED_IN_ELECTRIC_NETWORK,
-    NO_AMMO,
-    NO_FUEL,
-    NO_INGREDIENTS,
-    NO_INPUT_FLUID,
-    NO_MINABLE_RESOURCES,
-    NO_MODULES_TO_TRANSMIT,
-    NO_POWER,
-    NO_RECIPE,
-    NO_RESEARCH_IN_PROGRESS,
-    OPENED_BY_CIRCUIT_NETWORK,
-    OUT_OF_LOGISTIC_NETWORK,
-    PREPARING_ROCKET_FOR_LAUNCH,
-    RECHARGING_AFTER_POWER_OUTAGE,
-    TURNED_OFF_DURING_DAYTIME,
-    WAITING_FOR_SOURCE_ITEMS,
-    WAITING_FOR_SPACE_IN_DESTINATION,
-    WAITING_FOR_TARGET_TO_BE_BUILT,
-    WAITING_FOR_TRAIN,
-    WAITING_TO_LAUNCH_ROCKET,
     WORKING,
+    NORMAL,
+    NO_POWER,
+    LOW_POWER,
+    NO_FUEL,
+    DISABLED_BY_CONTROL_BEHAVIOR,
+    OPENED_BY_CIRCUIT_NETWORK,
+    CLOSED_BY_CIRCUIT_NETWORK,
+    DISABLED_BY_SCRIPT,
+    MARKED_FOR_DECONSTRUCTION,
+
+    /** Used by generators and solar panels. */
+    NOT_PLUGGED_IN_ELECTRIC_NETWORK,
+
+    /** Used by power switches. */
+    NETWORKS_CONNECTED,
+
+    /** Used by power switches. */
+    NETWORKS_DISCONNECTED,
+
+    /** Used by accumulators. */
+    CHARGING,
+    /** Used by accumulators. */
+    DISCHARGING,
+    /** Used by accumulators. */
+    FULLY_CHARGED,
+
+    /** Used by logistic containers. */
+    OUT_OF_LOGISTIC_NETWORK,
+
+    /** Used by assembling machines. */
+    NO_RECIPE,
+
+    /** Used by furnaces. */
+    NO_INGREDIENTS,
+
+    /** USED BY BOILERS, fluid turrets and fluid energy sources: Boiler has no fluid to work with. */
+    NO_INPUT_FLUID,
+
+    /** Used by labs. */
+    NO_RESEARCH_IN_PROGRESS,
+
+    /** Used by mining drills. */
+    NO_MINABLE_RESOURCES,
+
+    /** Used by boilers and fluid turrets: Boiler still has some fluid but is about to run out. */
+    LOW_INPUT_FLUID,
+
+    /** Used by crafting machines. */
+    FLUID_INGREDIENT_SHORTAGE,
+
+    /**
+     * Used by crafting machines, boilers, burner energy sources and reactors: Reactor/burner has
+     * full burnt result inventory, boiler has full output fluid box.
+     */
+    FULL_OUTPUT,
+
+    /** Used by crafting machines. */
+    ITEM_INGREDIENT_SHORTAGE,
+
+    /** Used by mining drills when the mining fluid is missing. */
+    MISSING_REQUIRED_FLUID,
+
+    /** Used by labs. */
+    MISSING_SCIENCE_PACKS,
+
+    /** Used by inserters. */
+    WAITING_FOR_SOURCE_ITEMS,
+
+    /** Used by inserters and mining drills. */
+    WAITING_FOR_SPACE_IN_DESTINATION,
+
+    /** Used by the rocket silo. */
+    PREPARING_ROCKET_FOR_LAUNCH,
+    /** Used by the rocket silo. */
+    WAITING_TO_LAUNCH_ROCKET,
+    /** Used by the rocket silo. */
+    LAUNCHING_ROCKET,
+
+    /** Used by beacons. */
+    NO_MODULES_TO_TRANSMIT,
+
+    /** Used by roboports. */
+    RECHARGING_AFTER_POWER_OUTAGE,
+
+    /** Used by inserters targeting entity ghosts. */
+    WAITING_FOR_TARGET_TO_BE_BUILT,
+    /** Used by inserters targeting rails. */
+    WAITING_FOR_TRAIN,
+
+    /** Used by ammo turrets. */
+    NO_AMMO,
+
+    /** Used by heat energy sources. */
+    LOW_TEMPERATURE,
+
+    /** Used by constant combinators: Combinator is turned off via switch in GUI. */
+    DISABLED,
+
+    /** Used by lamps. */
+    TURNED_OFF_DURING_DAYTIME,
+
+    /** Used by rail signals. */
+    NOT_CONNECTED_TO_RAIL,
+    /** Used by rail signals. */
+    CANT_DIVIDE_SEGMENTS,
   }
 
 }
-/*
-  enum entity_status {
-    working,
-    normal,
-    no_power,
-    low_power,
-    no_fuel,
-    disabled_by_control_behavior,
-    opened_by_circuit_network,
-    closed_by_circuit_network,
-    disabled_by_script,
-    marked_for_deconstruction,
-    not_plugged_in_electric_network,
-    networks_connected,
-    networks_disconnected,
-    charging,
-    discharging,
-    fully_charged,
-    out_of_logistic_network,
-    no_recipe,
-    no_ingredients,
-    no_input_fluid,
-    no_research_in_progress,
-    no_minable_resources,
-    low_input_fluid,
-    fluid_ingredient_shortage,
-    full_output,
-    item_ingredient_shortage,
-    missing_required_fluid,
-    missing_science_packs,
-    waiting_for_source_items,
-    waiting_for_space_in_destination,
-    preparing_rocket_for_launch,
-    waiting_to_launch_rocket,
-    launching_rocket,
-    no_modules_to_transmit,
-    recharging_after_power_outage,
-    waiting_for_target_to_be_built,
-    waiting_for_train,
-    no_ammo,
-    low_temperature,
-    disabled,
-    turned_off_during_daytime,
-    not_connected_to_rail,
-    cant_divide_segments,
+
+
+sealed interface FactorioEntityUpdateDictionary<T : FactorioEntityUpdateElement>
+  : Iterable<Pair<MapEntityPosition, T>> {
+
+  val entitiesXY: Map<String, Map<String, T>>
+
+  operator fun get(position: MapEntityPosition): T? =
+    entitiesXY[position.x.toString()]?.get(position.y.toString())
+
+  override fun iterator(): Iterator<Pair<MapEntityPosition, T>> =
+    iterator {
+      entitiesXY.forEach { (x, column) ->
+        column.forEach { (y, entity) ->
+          yield(
+            MapEntityPosition(x.toDouble(), y.toDouble()) to entity
+          )
+        }
+      }
     }
 
- */
+  @Serializable
+  @SerialName("kafkatorio.entity.FactorioEntityUpdateResourceDictionary")
+  @JvmInline
+  value class Resource(
+    override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Resource>>
+  ) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Resource>
+
+  @Serializable
+  @SerialName("kafkatorio.entity.FactorioEntityUpdateEntityDictionary")
+  @JvmInline
+  value class Entity(
+    override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Entity>>
+  ) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Entity>
+
+}
+
+
+sealed interface FactorioEntityUpdateElement {
+
+  @Serializable
+  @SerialName("kafkatorio.entity.FactorioEntityUpdateResource")
+  data class Resource(
+    val amount: UInt,
+    /** Count of initial resource units contained. `null` if resource is not infinite. */
+    val initialAmount: UInt? = null,
+  ) : FactorioEntityUpdateElement
+
+  @Serializable
+  @SerialName("kafkatorio.entity.FactorioEntityUpdateEntity")
+  data class Entity(
+    val status: FactorioEntityData.Status? = null,
+
+    val unitNumber: UnitNumber? = null,
+
+    val graphicsVariation: UByte? = null,
+    val health: Float? = null,
+    val isActive: Boolean? = null,
+    val isRotatable: Boolean? = null,
+    val lastUser: UInt? = null,
+  ) : FactorioEntityUpdateElement
+
+}
+
 
 /*
 AssemblingMachine

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioPrototypes.kt

@@ -1,6 +1,6 @@
 package dev.adamko.kafkatorio.schema.common
 
-import kotlinx.serialization.Contextual
+import dev.adamko.kafkatorio.library.LuaJsonList
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
@@ -20,8 +20,8 @@ sealed interface FactorioPrototype {
     val mapColour: Colour,
 
     val layer: UInt,
-    @Contextual
-    val collisionMasks: List<String>,
+//    @Contextual
+    val collisionMasks: LuaJsonList<String>,
     val order: String,
     /** Can the tile be mined for resources? */
     val canBeMined: Boolean,
@@ -49,17 +49,23 @@ sealed interface FactorioPrototype {
     val mapColourEnemy: Colour? = null,
 
     val maxHealth: Float,
-    val isBuilding: Boolean,
-    val isEntityWithOwner: Boolean,
-    val isMilitaryTarget: Boolean,
+    val isBuilding: Boolean = false,
+    val isEntityWithOwner: Boolean = false,
+    val isMilitaryTarget: Boolean = false,
     val miningProperties: MiningProperties? = null,
+
+    val collisionBox: MapBoundingBox? = null,
   ) : FactorioPrototype {
 
+    val tileHeight: Int? get() = collisionBox?.tileHeight
+    val tileWidth: Int? get() = collisionBox?.tileWidth
+
     @Serializable
     @SerialName("kafkatorio.prototype.EntityMiningProperties")
     data class MiningProperties(
       val canBeMined: Boolean,
-      val products: List<MinedProduct>?,
+//      @Contextual
+      val products: LuaJsonList<MinedProduct>?,
     )
 
     @Serializable

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/mapTile.kt

@@ -1,10 +1,11 @@
 package dev.adamko.kafkatorio.schema.common
 
-
 import dev.adamko.kafkatorio.schema.common.MapTileDictionary.PrototypeKey
 import kotlin.jvm.JvmInline
+import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
+
 const val MAP_CHUNK_SIZE = 32
 
 
@@ -24,11 +25,11 @@ data class MapTile(
  * [PrototypeKey] is an arbitrary integer value, which maps to one [PrototypeId] only for a
  * specific instance of [MapTileDictionary].
  *
- * Each X/Y coordinate in [tilesXY] maps to a [PrototypeKey].
+ * @param[tilesXY] Map an X/Y coordinate to a [PrototypeKey].
  */
 @Serializable
+@SerialName("kafkatorio.packet.keyed.MapTileDictionary")
 data class MapTileDictionary(
-  /** Map an X,Y coordinate a prototype name */
   val tilesXY: Map<String, Map<String, PrototypeKey>>,
   val protos: Map<PrototypeId, PrototypeKey>,
 ) {
@@ -55,59 +56,68 @@ data class MapTileDictionary(
   /** An arbitrary ID for a prototype in [MapTileDictionary.protos]. */
   @Serializable
   @JvmInline
-  value class PrototypeKey(private val index: Int)
+  value class PrototypeKey(private val index: Int) : Comparable<Int> by index
 
-  companion object {
-    fun MapTileDictionary?.isNullOrEmpty(): Boolean {
-      return this == null || (tilesXY.isEmpty() && tilesXY.all { it.value.isEmpty() })
-    }
-  }
+//  companion object {
+//    fun MapTileDictionary?.isNullOrEmpty(): Boolean {
+//      return this == null || (tilesXY.isEmpty() && tilesXY.all { it.value.isEmpty() })
+//    }
+//  }
 }
 
 
-@Serializable
-data class MapBoundingBox(
-  val topLeft: MapTilePosition,
-  val bottomRight: MapTilePosition,
-)
-
-
 @Serializable
 data class ServerMapChunkId(
   val serverId: FactorioServerId,
+  val layer: ServerMapTileLayer,
   val chunkPosition: MapChunkPosition,
   val surfaceIndex: SurfaceIndex,
-  val chunkSize: ChunkSize,
-)
+) {
+  val chunkSize: ChunkSize get() = chunkPosition.chunkSize
+}
 
 
-/** `${serverDataDir}/servers/{serverId}/map/tiles/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png` */
+@Serializable
+enum class ServerMapTileLayer(val dir: String) {
+  @SerialName("terrain")
+  TERRAIN("terrain"),
+  @SerialName("resource")
+  RESOURCE("resource"),
+  @SerialName("building")
+  BUILDING("building"),
+}
+
+
+/** `${serverDataDir}/servers/{serverId}/map/${layer}/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png` */
 @Serializable
 @JvmInline
-value class TilePngFilename(
-  val value: String,
-) {
+value class ServerMapTilePngFilename(val value: String) {
+
   constructor(id: ServerMapChunkId) : this(
-    id.serverId,
-    id.surfaceIndex,
-    id.chunkSize.zoomLevel,
-    id.chunkPosition.x,
-    id.chunkPosition.y,
+    serverId = id.serverId,
+    layer = id.layer,
+    surfaceIndex = id.surfaceIndex,
+    zoomLevel = id.chunkSize.zoomLevel,
+    chunkX = id.chunkPosition.x,
+    chunkY = id.chunkPosition.y,
   )
 
   constructor(
     serverId: FactorioServerId,
+    layer: ServerMapTileLayer,
     surfaceIndex: SurfaceIndex,
     zoomLevel: Int,
     chunkX: Int,
     chunkY: Int,
-  ) : this(buildString {
-    append("servers/${serverId}")
-    append("/map/tiles")
-    append("/s${surfaceIndex}")
-    append("/z${zoomLevel}")
-    append("/x${chunkX}")
-    append("/y${chunkY}")
-    append(".png")
-  })
+  ) : this(
+    "servers/${serverId}" +
+        "/map" +
+        "/layers/${layer.dir}" +
+        "/s${surfaceIndex}" +
+        "/z${zoomLevel}" +
+        "/x${chunkX}" +
+        "/y${chunkY}" +
+        ".png"
+  )
+
 }

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/positions.kt

@@ -1,23 +1,49 @@
 package dev.adamko.kafkatorio.schema.common
 
-import dev.adamko.kxstsgen.core.*
+import dev.adamko.kafkatorio.library.leftTopTile
+import dev.adamko.kafkatorio.library.rightBottomTile
 import dev.adamko.kxstsgen.core.experiments.TupleSerializer
-import kotlin.math.floor
+import kotlin.math.abs
+import kotlin.math.ceil
 import kotlin.math.pow
 import kotlin.math.roundToInt
+import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
+
 /**
- * Coordinates of a [EntityData] on a map.
+ * The position of something that exists in-game (not conceptually, like a
+ * [chunk][MapChunkPosition])
+ */
+sealed interface MapPosition : Comparable<MapPosition> {
+  val x: Number
+  val y: Number
+}
+
+
+/**
+ * Coordinates of an [entity][FactorioEntityData.Standard] on a map.
+ *
+ * Positive y points south, positive x points east.
  *
  * A [MapEntityPosition] can be translated to a [MapChunkPosition] by dividing the `x`/`y` values
  * by [32][MAP_CHUNK_SIZE].
  */
-@Serializable(with = MapEntityPosition.Serializer::class)
+@Serializable(with = MapEntityPosition.Companion.Serializer::class)
 data class MapEntityPosition(
-  val x: Double,
-  val y: Double,
-) {
+  override val x: Double,
+  override val y: Double,
+) : MapPosition {
+
+
+  override fun compareTo(other: MapPosition): Int =
+    when (val resultX = x.compareTo(other.x.toDouble())) {
+      0    -> y.compareTo(other.y.toDouble())
+      else -> resultX
+    }
+
+
+  companion object {
     object Serializer : TupleSerializer<MapEntityPosition>(
       "MapEntityPosition",
       {
@@ -32,73 +58,33 @@ data class MapEntityPosition(
         )
       }
     }
-}
-
-
-/** Coordinates of a chunk in a [SurfaceData] where each integer `x`/`y` represents a different
- * chunk.
- *
- * A [MapChunkPosition] can be translated to a [MapEntityPosition] by multiplying the `x`/`y`
- * values by [32][MAP_CHUNK_SIZE].
- */
-@Serializable(with = MapChunkPosition.Serializer::class)
-data class MapChunkPosition(
-  val x: Int,
-  val y: Int,
-) {
-  object Serializer : TupleSerializer<MapChunkPosition>(
-    "MapChunkPosition",
-    {
-      element(MapChunkPosition::x)
-      element(MapChunkPosition::y)
   }
-  ) {
-    override fun tupleConstructor(elements: Iterator<*>): MapChunkPosition {
-      return MapChunkPosition(
-        elements.next() as Int,
-        elements.next() as Int,
-      )
-    }
-  }
-
-  private val bounds: MapChunkPositionBounds by lazy {
-    MapChunkPositionBounds(this, ChunkSize.CHUNK_032)
-  }
-
-  operator fun contains(tilePosition: MapTilePosition): Boolean = tilePosition in bounds
-}
-
-
-@Serializable
-data class MapChunkPositionBounds(
-  val leftTop: MapTilePosition,
-  val rightBottom: MapTilePosition,
-) : ClosedRange<MapTilePosition> {
-
-  constructor(chunkPosition: MapChunkPosition, chunkSize: ChunkSize) : this(
-    chunkPosition.leftTopTile(chunkSize),
-    chunkPosition.rightBottomTile(chunkSize),
-  )
-
-  override val start: MapTilePosition
-    get() = leftTop
-
-  override val endInclusive: MapTilePosition
-    get() = rightBottom
 }
 
 
 /**
- * Coordinates of a tile in a chunk on a [SurfaceData] where each integer `x`/`y` represents a
- * different [MapTile].
+ * Coordinates of a tile on a surface, where each `x`/`y` coordinate represents a different
+ * [MapTile].
+ *
+ * Positive y points south, positive x points east.
  *
  * It rounds any `x`/`y` down to whole numbers.
  */
-@Serializable(with = MapTilePosition.Serializer::class)
+@Serializable(with = MapTilePosition.Companion.Serializer::class)
 data class MapTilePosition(
-  val x: Int,
-  val y: Int,
-) : Comparable<MapTilePosition> {
+  override val x: Int,
+  override val y: Int,
+) : MapPosition {
+
+
+  override fun compareTo(other: MapPosition): Int =
+    when (val resultX = x.compareTo(other.x.toInt())) {
+      0    -> y.compareTo(other.y.toInt())
+      else -> resultX
+    }
+
+
+  companion object {
     object Serializer : TupleSerializer<MapTilePosition>(
       "MapTilePosition",
       {
@@ -112,91 +98,82 @@ data class MapTilePosition(
         return MapTilePosition(x, y)
       }
     }
-
-  override operator fun compareTo(other: MapTilePosition): Int =
-    when {
-      other.x != x -> x.compareTo(other.x)
-      else         -> y.compareTo(other.y)
   }
+
 }
 
 
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
+/**
+ * Coordinates of a chunk on a surface, where each `x`/`y` coordinate represents a different
+ * chunk.
+ *
+ * Positive y points south, positive x points east.
+ *
+ * A [MapChunkPosition] can be translated to a [MapEntityPosition] by multiplying the `x`/`y`
+ * values by [32][MAP_CHUNK_SIZE].
+ */
+@Serializable(with = MapChunkPosition.Companion.Serializer::class)
+data class MapChunkPosition(
+  val x: Int,
+  val y: Int,
+  // Don't set a default for ChunkSize. It could default to 32, but it's clearer to make it explicit
+  // and avoid accidentally not setting it. Also, a default doesn't save much in serialization size.
+  val chunkSize: ChunkSize,
+) {
 
+  val bounds: MapChunkPositionBounds by lazy {
+    MapChunkPositionBounds(this)
+  }
 
-operator fun MapTilePosition.times(factor: Int) =
-  MapTilePosition(x * factor, y * factor)
+  operator fun contains(tilePosition: MapTilePosition): Boolean = tilePosition in bounds
+  operator fun contains(entityPosition: MapEntityPosition): Boolean = entityPosition in bounds
 
-operator fun MapTilePosition.div(divisor: Int) =
-  MapTilePosition(x / divisor, y / divisor)
-
-operator fun MapTilePosition.plus(addend: Int) =
-  MapTilePosition(x + addend, y + addend)
-
-operator fun MapTilePosition.plus(addend: MapTilePosition) =
-  MapTilePosition(x + addend.x, y + addend.y)
-
-
-operator fun MapTilePosition.minus(subtrahend: MapTilePosition) =
-  MapTilePosition(x - subtrahend.x, y - subtrahend.y)
-
-
-fun MapTilePosition.toMapChunkPosition(chunkSize: ChunkSize) =
-  MapChunkPosition(
-    floor(x.toDouble() / chunkSize.lengthInTiles.toDouble()).toInt(),
-    floor(y.toDouble() / chunkSize.lengthInTiles.toDouble()).toInt(),
+  companion object {
+    object Serializer : TupleSerializer<MapChunkPosition>(
+      "MapChunkPosition",
+      {
+        element(MapChunkPosition::x)
+        element(MapChunkPosition::y)
+        element(MapChunkPosition::chunkSize)
+      }
+    ) {
+      override fun tupleConstructor(elements: Iterator<*>): MapChunkPosition {
+        return MapChunkPosition(
+          elements.next() as Int,
+          elements.next() as Int,
+          elements.next() as ChunkSize,
         )
-
-
-fun MapEntityPosition.toMapTilePosition() =
-  MapChunkPosition(floor(x).toInt(), floor(y).toInt())
-
-fun MapEntityPosition.toMapChunkPosition(chunkSize: ChunkSize) =
-  MapChunkPosition(
-    floor(x / chunkSize.lengthInTiles.toDouble()).toInt(),
-    floor(y / chunkSize.lengthInTiles.toDouble()).toInt(),
-  )
-
-
-operator fun MapChunkPosition.times(factor: Int) =
-  MapChunkPosition(x * factor, y * factor)
-
-operator fun MapChunkPosition.plus(addend: Int) =
-  MapChunkPosition(x + addend, y + addend)
-
-operator fun MapChunkPosition.minus(subtrahend: Int) =
-  MapChunkPosition(x - subtrahend, y - subtrahend)
-
-
-fun MapChunkPosition.leftTopTile(chunkSize: ChunkSize): MapTilePosition =
-  MapTilePosition(
-    x * chunkSize.lengthInTiles,
-    y * chunkSize.lengthInTiles,
-  )
-
-fun MapChunkPosition.rightBottomTile(chunkSize: ChunkSize): MapTilePosition =
-  leftTopTile(chunkSize) + (chunkSize.lengthInTiles - 1)
-
-
-fun MapChunkPosition.iterateTiles(chunkSize: ChunkSize): Iterator<MapTilePosition> {
-  val leftTop = leftTopTile(chunkSize)
-  val rightBottom = rightBottomTile(chunkSize)
-  return iterator {
-    (leftTop.x..rightBottom.x).forEach { x ->
-      (leftTop.y..rightBottom.y).forEach { y ->
-        yield(MapTilePosition(x, y))
       }
     }
   }
 }
 
 
-//val MapChunkPosition.rightBottomTile: MapTilePosition
-//  get() = leftTopTile + (MAP_CHUNK_SIZE - 1)
+/**
+ * The bounds of a [MapChunkPosition], that may contain a [MapPosition].
+ *
+ * Because Factorio uses screen-coordinates, the lowest `x/y` coordinate is the [leftTop], and
+ * the highest is [rightBottomTile].
+ */
+@Serializable
+data class MapChunkPositionBounds(
+  val leftTop: MapTilePosition,
+  val rightBottom: MapTilePosition,
+  val chunkSize: ChunkSize, // should be computable based on the tiles, but include it for completeness
+) : ClosedRange<MapPosition> {
 
-//
-//val MapChunkPosition.boundingBox: MapBoundingBox
-//  get() = MapBoundingBox(leftTopTile, rightBottomTile)
+  constructor(chunkPosition: MapChunkPosition) : this(
+    chunkPosition.leftTopTile(),
+    chunkPosition.rightBottomTile(),
+    chunkPosition.chunkSize,
+  )
+
+  override val start: MapTilePosition
+    get() = leftTop
+
+  override val endInclusive: MapTilePosition
+    get() = rightBottom
+}
 
 
 @Serializable
@@ -231,3 +208,17 @@ enum class ChunkSize(
     val STANDARD: ChunkSize = entries.first { it.zoomLevel == 0 }
   }
 }
+
+
+@Serializable
+@SerialName("kafkatorio.position.MapBoundingBox")
+data class MapBoundingBox(
+  val leftTop: MapEntityPosition,
+  val rightBottom: MapEntityPosition,
+) {
+  val width: Double get() = abs(leftTop.x - rightBottom.x)
+  val height: Double get() = abs(leftTop.y - rightBottom.y)
+
+  val tileWidth: Int get() = ceil(width).toInt()
+  val tileHeight: Int get() = ceil(height).toInt()
+}

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/prototypeIds.kt

@@ -55,6 +55,15 @@ value class PrototypeId private constructor(
 }
 
 
+/** Serialize a [PrototypeId] as an integer - more efficient storage */
+@JvmInline
+@Serializable
+value class PrototypeHashCode private constructor(val code: Int) {
+  constructor(id: PrototypeId) : this(id.hashCode())
+  constructor(tile: FactorioPrototype) : this(tile.protoId)
+}
+
+
 //@Suppress("unused")
 //enum class PrototypeTypeId(
 //  val id: String

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/eventServerPacket.kt

@@ -1,7 +1,7 @@
 package dev.adamko.kafkatorio.schema.packets
 
 import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
-import dev.adamko.kafkatorio.schema.common.TilePngFilename
+import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
@@ -21,7 +21,7 @@ sealed class EventServerPacket {
   @SerialName("kafkatorio.event-server-packet.ChunkTileSaved")
   data class ChunkTileSaved(
     val chunkId: ServerMapChunkId,
-    val filename: TilePngFilename,
+    val filename: ServerMapTilePngFilename,
   ) : EventServerPacket()
 
 }

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt

@@ -16,13 +16,12 @@ data class KafkatorioPacket(
 
 @Serializable
 @SerialName("kafkatorio.packet.KafkatorioPacketData")
-sealed class KafkatorioPacketData {
+sealed class KafkatorioPacketData
 
-  @Serializable
-  @SerialName("kafkatorio.packet.KafkatorioPacketData.Error")
-  data class Error(
+
+@Serializable
+@SerialName("kafkatorio.packet.KafkatorioPacketData.Error")
+data class KafkatorioPacketDataError(
   val message: String? = null,
   val rawValue: String? = null,
-  ) : KafkatorioPacketData()
-
-}
+) : KafkatorioPacketData()

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketInstant.kt

@@ -1,5 +1,6 @@
 package dev.adamko.kafkatorio.schema.packets
 
+import dev.adamko.kafkatorio.library.LuaJsonList
 import dev.adamko.kafkatorio.schema.common.FactorioPrototype
 import dev.adamko.kafkatorio.schema.common.PlayerIndex
 import dev.adamko.kafkatorio.schema.common.SurfaceIndex
@@ -20,7 +21,8 @@ sealed class KafkatorioInstantPacketData : KafkatorioPacketData()
 @SerialName("kafkatorio.packet.instant.ConfigurationUpdate")
 data class ConfigurationUpdate(
   val factorioData: ConfigurationUpdateGameData,
-  val allMods: List<ConfigurationUpdateModData>,
+//  @Contextual
+  val allMods: LuaJsonList<ConfigurationUpdateModData>,
   val modStartupSettingsChange: Boolean,
   val migrationApplied: Boolean,
 ) : KafkatorioInstantPacketData() {
@@ -62,7 +64,8 @@ data class ConsoleCommandUpdate(
 @Serializable
 @SerialName("kafkatorio.packet.instant.PrototypesUpdate")
 data class PrototypesUpdate(
-  val prototypes: List<FactorioPrototype>,
+//  @Contextual
+  val prototypes: LuaJsonList<FactorioPrototype>,
 ) : KafkatorioInstantPacketData()
 
 

modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketKeyed.kt

@@ -1,10 +1,12 @@
 package dev.adamko.kafkatorio.schema.packets
 
+import dev.adamko.kafkatorio.library.LuaJsonList
 import dev.adamko.kafkatorio.schema.common.Colour
 import dev.adamko.kafkatorio.schema.common.EntityIdentifiers
 import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
 import dev.adamko.kafkatorio.schema.common.EventName
 import dev.adamko.kafkatorio.schema.common.FactorioEntityData
+import dev.adamko.kafkatorio.schema.common.FactorioEntityUpdateDictionary
 import dev.adamko.kafkatorio.schema.common.ForceIndex
 import dev.adamko.kafkatorio.schema.common.MapChunkPosition
 import dev.adamko.kafkatorio.schema.common.MapEntityPosition
@@ -26,7 +28,7 @@ sealed class KafkatorioKeyedPacketData : KafkatorioPacketData() {
   /** A list of all events that have been aggregated into this packet. */
   // Maybe change to be Map<EventName, Tick>, where Tick is the most recent event?
   // A complete event timeline isn't necessary. The most recent tick of an event is good enough, and requires less space.
-  abstract val events: Map<EventName, List<Tick>>?
+  abstract val events: Map<EventName, LuaJsonList<Tick>>?
 }
 
 
@@ -52,13 +54,13 @@ data class EntityUpdateKey(
 data class EntityUpdate(
   override val key: EntityUpdateKey,
 
-  override val events: Map<EventName, List<Tick>>? = null,
+  override val events: Map<EventName, LuaJsonList<Tick>>? = null,
 
   val chunkPosition: MapEntityPosition? = null,
   val graphicsVariation: UByte? = null,
   val health: Float? = null,
-  val isActive: Boolean? = null,
-  val isRotatable: Boolean? = null,
+  val isActive: Boolean = false,
+  val isRotatable: Boolean = false,
   val lastUser: UInt? = null,
   val localisedDescription: String? = null,
   val localisedName: String? = null,
@@ -82,14 +84,14 @@ data class MapChunkTileUpdateKey(
 data class MapChunkTileUpdate(
   override val key: MapChunkTileUpdateKey,
 
-  override val events: Map<EventName, List<Tick>>? = null,
+  override val events: Map<EventName, LuaJsonList<Tick>>? = null,
 
   val player: PlayerIndex? = null,
   val robot: EntityIdentifiersData? = null,
   val force: ForceIndex? = null,
   /** updated tiles - might be partial and not all tiles in the chunk */
   val tileDictionary: MapTileDictionary? = null,
-  val isDeleted: Boolean? = null,
+  val isDeleted: Boolean = false,
 ) : KafkatorioKeyedPacketData()
 
 
@@ -108,7 +110,7 @@ data class PlayerUpdateKey(
 data class PlayerUpdate(
   override val key: PlayerUpdateKey,
 
-  override val events: Map<EventName, List<Tick>>? = null,
+  override val events: Map<EventName, LuaJsonList<Tick>>? = null,
 
   val characterUnitNumber: UnitNumber? = null,
   val chatColour: Colour? = null,
@@ -118,10 +120,10 @@ data class PlayerUpdate(
   val afkTime: Tick? = null,
   val ticksToRespawn: Tick? = null,
   val forceIndex: ForceIndex? = null,
-  val isAdmin: Boolean? = null,
-  val isConnected: Boolean? = null,
-  val isShowOnMap: Boolean? = null,
-  val isSpectator: Boolean? = null,
+  val isAdmin: Boolean = false,
+  val isConnected: Boolean = false,
+  val isShowOnMap: Boolean = false,
+  val isSpectator: Boolean = false,
   val lastOnline: Tick? = null,
   val onlineTime: Tick? = null,
   val position: MapEntityPosition? = null,
@@ -149,20 +151,58 @@ data class MapChunkEntityUpdateKey(
 ) : KafkatorioKeyedPacketKey()
 
 
-/**
- * @property[distinctEntities] Distinct entities. The key is arbitrary and is only used
- * for de-duplicating updates. It should only contain data that can be obtained from
- * [the value][FactorioEntityData].
- */
 @Serializable
 @SerialName("kafkatorio.packet.keyed.MapChunkEntityUpdate")
 data class MapChunkEntityUpdate(
   override val key: MapChunkEntityUpdateKey,
 
-  override val events: Map<EventName, List<Tick>>? = null,
+  override val events: Map<EventName, LuaJsonList<Tick>>? = null,
 
-  val distinctEntities: Map<String, FactorioEntityData>? = null,
-) : KafkatorioKeyedPacketData()
+  val entitiesXY: FactorioEntityUpdateDictionary.Entity,
+) : KafkatorioKeyedPacketData() {
+
+  fun entities(): List<FactorioEntityData.Standard> =
+    entitiesXY.map { (position, entity) ->
+      FactorioEntityData.Standard(
+        protoId = key.protoId,
+        position = position,
+        element = entity,
+      )
+    }
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+@SerialName("kafkatorio.packet.keyed.MapChunkResourceUpdateKey")
+data class MapChunkResourceUpdateKey(
+  val protoId: PrototypeId,
+  val surfaceIndex: SurfaceIndex,
+  val chunkPosition: MapChunkPosition,
+) : KafkatorioKeyedPacketKey()
+
+
+@Serializable
+@SerialName("kafkatorio.packet.keyed.MapChunkResourceUpdate")
+data class MapChunkResourceUpdate(
+  override val key: MapChunkResourceUpdateKey,
+
+  override val events: Map<EventName, LuaJsonList<Tick>>? = null,
+
+  val resourcesXY: FactorioEntityUpdateDictionary.Resource,
+) : KafkatorioKeyedPacketData() {
+
+  fun resources(): List<FactorioEntityData.Resource> =
+    resourcesXY.map { (position, resource) ->
+      FactorioEntityData.Resource(
+        protoId = key.protoId,
+        position = position,
+        element = resource,
+      )
+    }
+}
 
 
 /*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */

modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/ColourHexTest.kt

@@ -1,10 +1,6 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema.common
 
 import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
-import dev.adamko.kafkatorio.schema.common.Colour
-import dev.adamko.kafkatorio.schema.common.ColourHex
-import dev.adamko.kafkatorio.schema.common.toHex
-import dev.adamko.kafkatorio.schema.common.toPercentile
 import io.kotest.assertions.withClue
 import io.kotest.core.spec.style.BehaviorSpec
 import io.kotest.matchers.floats.shouldBeWithinPercentageOf

modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapChunkPositionBoundsTest.kt

@@ -2,10 +2,12 @@ package dev.adamko.kafkatorio.schema.common
 
 import dev.adamko.kafkatorio.library.kxsBinary
 import io.kotest.assertions.asClue
+import io.kotest.assertions.withClue
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.collections.shouldContainExactly
 import io.kotest.matchers.ints.shouldBeExactly
 import io.kotest.matchers.maps.shouldContainExactly
+import io.kotest.matchers.shouldBe
 import kotlinx.serialization.decodeFromByteArray
 import kotlinx.serialization.encodeToByteArray
 
@@ -17,18 +19,17 @@ class MapChunkPositionBoundsTest : FunSpec({
 
       context("chunk size 32") {
 
-        val chunkPos = MapChunkPosition(0, 0)
-        val bounds = MapChunkPositionBounds(chunkPos, ChunkSize.CHUNK_032)
+        val chunkPos = MapChunkPosition(0, 0, ChunkSize.CHUNK_032)
 
         test("leftTop should be 0,0") {
-          bounds.leftTop.asClue {
+          chunkPos.bounds.leftTop.asClue {
             it.x shouldBeExactly 0
             it.y shouldBeExactly 0
           }
         }
 
         test("rightBottom should be 31,31") {
-          bounds.rightBottom.asClue {
+          chunkPos.bounds.rightBottom.asClue {
             it.x shouldBeExactly 31
             it.y shouldBeExactly 31
           }
@@ -38,18 +39,17 @@ class MapChunkPositionBoundsTest : FunSpec({
 
     context("chunk size 256") {
 
-      val chunkPos = MapChunkPosition(0, 0)
-      val bounds = MapChunkPositionBounds(chunkPos, ChunkSize.CHUNK_256)
+      val chunkPos = MapChunkPosition(0, 0, ChunkSize.CHUNK_256)
 
       test("leftTop should be 0,0") {
-        bounds.leftTop.asClue {
+        chunkPos.bounds.leftTop.asClue {
           it.x shouldBeExactly 0
           it.y shouldBeExactly 0
         }
       }
 
       test("rightBottom should be 255,255") {
-        bounds.rightBottom.asClue {
+        chunkPos.bounds.rightBottom.asClue {
           it.x shouldBeExactly 255
           it.y shouldBeExactly 255
         }
@@ -61,18 +61,17 @@ class MapChunkPositionBoundsTest : FunSpec({
 
     context("chunk size 32") {
 
-      val chunkPos = MapChunkPosition(-10, -10)
-      val bounds = MapChunkPositionBounds(chunkPos, ChunkSize.CHUNK_032)
+      val chunkPos = MapChunkPosition(-10, -10, ChunkSize.CHUNK_032)
 
       test("leftTop should be -320,-320") {
-        bounds.leftTop.asClue {
+        chunkPos.bounds.leftTop.asClue {
           it.x shouldBeExactly -320
           it.y shouldBeExactly -320
         }
       }
 
       test("rightBottom should be -289,-289") {
-        bounds.rightBottom.asClue {
+        chunkPos.bounds.rightBottom.asClue {
           it.x shouldBeExactly -289
           it.y shouldBeExactly -289
         }
@@ -81,18 +80,17 @@ class MapChunkPositionBoundsTest : FunSpec({
 
     context("chunk size 256") {
 
-      val chunkPos = MapChunkPosition(-10, -10)
-      val bounds = MapChunkPositionBounds(chunkPos, ChunkSize.CHUNK_256)
+      val chunkPos = MapChunkPosition(-10, -10, ChunkSize.CHUNK_256)
 
       test("leftTop should be -2560") {
-        bounds.leftTop.asClue {
+        chunkPos.bounds.leftTop.asClue {
           it.x shouldBeExactly -2560
           it.y shouldBeExactly -2560
         }
       }
 
       test("rightBottom should be (-2560 + 255)") {
-        bounds.rightBottom.asClue {
+        chunkPos.bounds.rightBottom.asClue {
           it.x shouldBeExactly (-2560 + 255)
           it.y shouldBeExactly (-2560 + 255)
         }
@@ -134,4 +132,57 @@ class MapChunkPositionBoundsTest : FunSpec({
       decoded.toMapTileList() shouldContainExactly actual.toMapTileList()
     }
   }
+
+  context("verify map chunk 'contains'") {
+
+    context("MapChunkPosition(1, 1)") {
+
+      val chunkPos = MapChunkPosition(1, 1, ChunkSize.CHUNK_032) // from 32,32 to 63,63
+
+      test("tile outside of chunk") {
+        val outerTilePos = MapTilePosition(1, 1)
+
+        withClue("$outerTilePos in ${chunkPos.bounds}") {
+          (outerTilePos in chunkPos) shouldBe false
+        }
+      }
+
+      test("tile inside of chunk") {
+        val innerTilePos = MapTilePosition(34, 33)
+
+        withClue("$innerTilePos in ${chunkPos.bounds}") {
+          (innerTilePos in chunkPos) shouldBe true
+        }
+      }
+
+      test("entity outside of chunk") {
+        val outerEntityPos = MapEntityPosition(1.0, 1.0)
+
+        withClue("$outerEntityPos in ${chunkPos.bounds}") {
+          (outerEntityPos !in chunkPos) shouldBe true
+          (outerEntityPos in chunkPos) shouldBe false
+        }
+      }
+
+      test("entity inside of chunk") {
+        val innerEntityPos = MapEntityPosition(34.0, 33.0)
+
+        withClue("$innerEntityPos in ${chunkPos.bounds}") {
+          (innerEntityPos !in chunkPos) shouldBe false
+          (innerEntityPos in chunkPos) shouldBe true
+        }
+      }
+    }
+    context(" MapChunkPosition(9, -5)") {
+      val chunkPos2 = MapChunkPosition(9, -5, ChunkSize.CHUNK_032)
+
+      test("tile inside of chunk") {
+        val innerTilePos = MapTilePosition(288, -160)
+
+        withClue("$innerTilePos in ${chunkPos2.bounds}") {
+          (innerTilePos in chunkPos2) shouldBe true
+        }
+      }
+    }
+  }
 })

modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapChunkPositionTest.kt

@@ -14,7 +14,7 @@ class MapChunkPositionTest : FunSpec({
 
     context("MapChunkPosition(0, 0)") {
 
-      val chunkPos = MapChunkPosition(0, 0)
+      val chunkPos = MapChunkPosition(0, 0, ChunkSize.CHUNK_032)
 
       test("should contain tiles from (0,0) to (31,31)") {
         checkAll(mapTilePositionArb(0..31)) { tilePos ->
@@ -40,7 +40,7 @@ class MapChunkPositionTest : FunSpec({
     }
     context("MapChunkPosition(-10, -10)") {
 
-      val chunkPos = MapChunkPosition(-10, -10)
+      val chunkPos = MapChunkPosition(-10, -10, ChunkSize.CHUNK_032)
 
       test("should contain tiles from (-320,-320) to (-289,-289)") {
         checkAll(mapTilePositionArb(-320..-289)) { tilePos ->

modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTilePositionTest.kt

@@ -1,5 +1,7 @@
 package dev.adamko.kafkatorio.schema.common
 
+import dev.adamko.kafkatorio.library.minus
+import dev.adamko.kafkatorio.library.plus
 import dev.adamko.kafkatorio.schema.common.MapChunkPositionTest.Companion.mapTilePositionArb
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.property.forAll
@@ -17,15 +19,27 @@ class MapTilePositionTest : FunSpec({
       }
     }
 
-    test("less than or equal to <=") {
+    test("equal comparing to ==") {
       forAll(mapTilePositionArb()) { tilePos ->
-        tilePos <= (tilePos)
+        tilePos.compareTo(tilePos) == 0
+      }
+    }
+
+    test("less than or equal to <=") {
+      forAll(
+        mapTilePositionArb(),
+        mapTilePositionArb(0..1),
+      ) { tilePos, delta ->
+        tilePos <= (tilePos + delta)
       }
     }
 
     test("greater than or equal to >=") {
-      forAll(mapTilePositionArb()) { tilePos ->
-        tilePos >= (tilePos)
+      forAll(
+        mapTilePositionArb(),
+        mapTilePositionArb(0..1),
+      ) { tilePos, delta ->
+        tilePos >= (tilePos - delta)
       }
     }
 
@@ -37,7 +51,5 @@ class MapTilePositionTest : FunSpec({
         tilePos > (tilePos - delta)
       }
     }
-
   }
-
 })

modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/PrototypesTest.kt

@@ -1,9 +1,6 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema.common
 
 import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
-import dev.adamko.kafkatorio.schema.common.Colour
-import dev.adamko.kafkatorio.schema.common.FactorioPrototype
-import dev.adamko.kafkatorio.schema.common.PrototypeId
 import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
 import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
 import io.kotest.core.spec.style.BehaviorSpec

modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/packets/MapChunkTileUpdateTest.kt

@@ -1,22 +1,21 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema.packets
 
+import dev.adamko.kafkatorio.library.LuaJsonList
 import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
+import dev.adamko.kafkatorio.schema.common.ChunkSize
 import dev.adamko.kafkatorio.schema.common.EventName
 import dev.adamko.kafkatorio.schema.common.MapChunkPosition
 import dev.adamko.kafkatorio.schema.common.MapTileDictionary
 import dev.adamko.kafkatorio.schema.common.PrototypeId
 import dev.adamko.kafkatorio.schema.common.SurfaceIndex
 import dev.adamko.kafkatorio.schema.common.tick
-import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
-import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
-import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdateKey
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.shouldBe
 import kotlinx.serialization.decodeFromString
 
-class MapChunkUpdateTest : FunSpec({
+class MapChunkTileUpdateTest : FunSpec({
 
-  context("MapChunkUpdate update") {
+  context("MapChunkTileUpdate update") {
     //language=json
     val json = """
 {
@@ -26,7 +25,8 @@ class MapChunkUpdateTest : FunSpec({
       "surfaceIndex": 1,
       "chunkPosition": [
         -6,
-        -3
+        -3,
+        "CHUNK_032"
       ]
     },
     "tileDictionary": {
@@ -61,11 +61,11 @@ class MapChunkUpdateTest : FunSpec({
       modVersion = "0.4.0",
       data = MapChunkTileUpdate(
         key = MapChunkTileUpdateKey(
-          MapChunkPosition(-6, -3),
+          MapChunkPosition(-6, -3, ChunkSize.CHUNK_032),
           SurfaceIndex(1u),
         ),
         events = mapOf(
-          EventName("on_chunk_generated") to listOf(1u.tick)
+          EventName("on_chunk_generated") to LuaJsonList(listOf(1u.tick))
         ),
         tileDictionary = MapTileDictionary(
           tilesXY = mapOf(

modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/packets/UpdatePacketsTest.kt

@@ -1,5 +1,6 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema.packets
 
+import dev.adamko.kafkatorio.library.LuaJsonList
 import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
 import dev.adamko.kafkatorio.schema.common.Colour
 import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
@@ -12,15 +13,12 @@ import dev.adamko.kafkatorio.schema.common.SurfaceIndex
 import dev.adamko.kafkatorio.schema.common.Tick
 import dev.adamko.kafkatorio.schema.common.UnitNumber
 import dev.adamko.kafkatorio.schema.common.tick
-import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
-import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
-import dev.adamko.kafkatorio.schema.packets.PlayerUpdateKey
-import dev.adamko.kafkatorio.schema.packets.SurfaceUpdate
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.matchers.shouldBe
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 
+
 class UpdatePacketsTest : FunSpec({
 
   context("player update packet") {
@@ -72,8 +70,8 @@ class UpdatePacketsTest : FunSpec({
           diedCause = null,
           disconnectReason = null,
           events = mapOf(
-            EventName("on_player_changed_position") to listOf(7u.tick, 99u.tick),
-            EventName("on_player_joined_game") to listOf(14u.tick),
+            EventName("on_player_changed_position") to LuaJsonList(listOf(7u.tick, 99u.tick)),
+            EventName("on_player_joined_game") to LuaJsonList(listOf(14u.tick)),
           ),
           forceIndex = ForceIndex(1u),
           isAdmin = true,
@@ -194,7 +192,7 @@ class UpdatePacketsTest : FunSpec({
           ),
           disconnectReason = null,
           events = mapOf(
-            EventName("on_player_died") to listOf(2u.tick),
+            EventName("on_player_died") to LuaJsonList(listOf(2u.tick)),
           ),
           forceIndex = null,
           key = PlayerUpdateKey(PlayerIndex(1u)),

modules/events-library/src/jvmMain/kotlin/dev/adamko/kafkatorio/schema/kt2ts2.kt

@@ -1,4 +1,4 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema
 
 import dev.adamko.kafkatorio.schema.common.PlayerIndex
 import dev.adamko.kafkatorio.schema.common.SurfaceIndex
@@ -85,13 +85,21 @@ class MySourceCodeGenerator(
   config: KxsTsConfig,
 ) : TsSourceCodeGenerator.Default(config) {
 
+  private val seenTypeAliases: MutableSet<TsElementId> = mutableSetOf()
+
   override fun generateTypeAlias(element: TsTypeAlias): String {
 
+    if (!seenTypeAliases.add(element.id)) {
+      return ""
+    }
+
     return if (element.id.name in listOf("PlayerIndex", "SurfaceIndex", "UnitNumber")) {
       ""
     } else if (
       (element.typeRef as? TsTypeRef.Literal)?.element is TsLiteral.Custom
       || element.id.name == "Tick"
+      || element.id.name == "FactorioEntityUpdateEntityDictionary"
+      || element.id.name == "FactorioEntityUpdateResourceDictionary"
     ) {
       val aliasedRef = generateTypeReference(element.typeRef)
       """

modules/events-library/src/jvmTest/kotlin/dev/adamko/kafkatorio/library/KxsBinaryTests.kt

@@ -1,11 +1,12 @@
 package dev.adamko.kafkatorio.library
 
-import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
 import dev.adamko.kafkatorio.schema.common.ChunkSize
 import dev.adamko.kafkatorio.schema.common.ColourHex
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
 import dev.adamko.kafkatorio.schema.common.MapChunkPosition
 import dev.adamko.kafkatorio.schema.common.MapTilePosition
+import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
 import dev.adamko.kafkatorio.schema.common.SurfaceIndex
 import io.kotest.assertions.withClue
 import io.kotest.core.spec.style.FunSpec
@@ -51,7 +52,7 @@ class KxsBinaryTests : FunSpec({
 
   test("byte array") {
     val ba = byteArrayOf(
-      4, 116, 101, 115, 116, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
+      4, 116, 101, 115, 116, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 1
     )
 
     val result = kxsBinary.decodeFromByteArray(ServerMapChunkId.serializer(), ba)
@@ -60,9 +61,9 @@ class KxsBinaryTests : FunSpec({
 
       result shouldBeEqualToComparingFields ServerMapChunkId(
         serverId = FactorioServerId("test"),
-        chunkPosition = MapChunkPosition(1, 2),
+        layer = ServerMapTileLayer.TERRAIN,
+        chunkPosition = MapChunkPosition(1, 2, ChunkSize.CHUNK_064),
         surfaceIndex = SurfaceIndex(1u),
-        chunkSize = ChunkSize.CHUNK_064,
       )
 
     }
@@ -72,15 +73,15 @@ class KxsBinaryTests : FunSpec({
     val ba = kxsBinary.encodeToByteArray(
       ServerMapChunkId(
         serverId = FactorioServerId("test"),
-        chunkPosition = MapChunkPosition(1, 2),
+        layer = ServerMapTileLayer.TERRAIN,
+        chunkPosition = MapChunkPosition(1, 2, ChunkSize.CHUNK_064),
         surfaceIndex = SurfaceIndex(1u),
-        chunkSize = ChunkSize.CHUNK_064,
       )
     )
 
     withClue(ba.toString(Charsets.UTF_8)) {
       ba shouldBe byteArrayOf(
-        4, 116, 101, 115, 116, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
+        4, 116, 101, 115, 116, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 1
       )
     }
   }
@@ -112,10 +113,10 @@ class KxsBinaryTests : FunSpec({
   context("ServerMapChunkTiles") {
     val initial = ServerMapChunkTiles(
       ServerMapChunkId(
-        FactorioServerId("server-id"),
-        MapChunkPosition(33, 44),
-        SurfaceIndex(4u),
-        ChunkSize.CHUNK_032,
+        serverId = FactorioServerId("server-id"),
+        layer = ServerMapTileLayer.TERRAIN,
+        chunkPosition = MapChunkPosition(33, 44, ChunkSize.CHUNK_032),
+        surfaceIndex = SurfaceIndex(4u),
       ),
       mapOf(
         MapTilePosition(1, 2) to ColourHex(1u, 2u, 3u, 4u),

modules/events-mod/build.gradle.kts

@@ -68,7 +68,11 @@ dependencies {
 
 
 val typescriptToLua by tasks.registering(TypescriptToLuaTask::class) {
-  dependsOn(tasks.npmInstall, installEventsTsSchema, tasks.updatePackageJson)
+  dependsOn(
+    tasks.npmInstall,
+    installEventsTsSchema,
+    tasks.updatePackageJson,
+  )
 
   inputs.file(tasks.updatePackageJson.map { it.packageJsonFile })
 

modules/events-mod/package-lock.json

@@ -85,9 +85,9 @@
       "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ=="
     },
     "typescript-to-lua": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/typescript-to-lua/-/typescript-to-lua-1.6.2.tgz",
-      "integrity": "sha512-m0TZ2RqL6BPy3/TNIjbVDTr9N6M45Qyb2/VJpBz0qnS3zTXxhAOugc+PaILSVwvCb1vent8ateWDt2fcEtcsRw==",
+      "version": "1.6.3",
+      "resolved": "https://registry.npmjs.org/typescript-to-lua/-/typescript-to-lua-1.6.3.tgz",
+      "integrity": "sha512-yfXxGjOwX6453wSgEB37lLdxZZ8m+LH1g3PfebNfKaZHGGtbm9owxE6oRKB3sWcy2CICkfobAjWf3VdhKnEETg==",
       "requires": {
         "enhanced-resolve": "^5.8.2",
         "resolve": "^1.15.1",

modules/events-mod/package.json

@@ -12,7 +12,7 @@
   "devDependencies": {},
   "dependencies": {
     "lua-types": "^2.11.0",
-    "typescript-to-lua": "1.6.2",
+    "typescript-to-lua": "1.6.3",
     "typed-factorio": "1.3.2",
     "typescript": "4.7.4"
   },

modules/events-mod/src/generated/lua/generated/kafkatorio-schema.lua

@@ -10,6 +10,7 @@ do
     KafkatorioPacketData.Type.SurfaceUpdate = "kafkatorio.packet.instant.SurfaceUpdate"
     KafkatorioPacketData.Type.EntityUpdate = "kafkatorio.packet.keyed.EntityUpdate"
     KafkatorioPacketData.Type.MapChunkEntityUpdate = "kafkatorio.packet.keyed.MapChunkEntityUpdate"
+    KafkatorioPacketData.Type.MapChunkResourceUpdate = "kafkatorio.packet.keyed.MapChunkResourceUpdate"
     KafkatorioPacketData.Type.MapChunkTileUpdate = "kafkatorio.packet.keyed.MapChunkTileUpdate"
     KafkatorioPacketData.Type.PlayerUpdate = "kafkatorio.packet.keyed.PlayerUpdate"
     KafkatorioPacketData.Type.Error = "kafkatorio.packet.KafkatorioPacketData.Error"
@@ -21,57 +22,12 @@ do
     FactorioPrototype.Type.Entity = "kafkatorio.prototype.Entity"
     FactorioPrototype.Type.MapTile = "kafkatorio.prototype.MapTile"
 end
-____exports.FactorioEntityData = {}
-local FactorioEntityData = ____exports.FactorioEntityData
-do
-    FactorioEntityData.Type = Type or ({})
-    FactorioEntityData.Type.Resource = "kafkatorio.entity.FactorioEntityData.Resource"
-    FactorioEntityData.Type.Standard = "kafkatorio.entity.FactorioEntityData.Standard"
-end
-____exports.EntityStatus = EntityStatus or ({})
-____exports.EntityStatus.CANT_DIVIDE_SEGMENTS = "CANT_DIVIDE_SEGMENTS"
-____exports.EntityStatus.CHARGING = "CHARGING"
-____exports.EntityStatus.CLOSED_BY_CIRCUIT_NETWORK = "CLOSED_BY_CIRCUIT_NETWORK"
-____exports.EntityStatus.DISABLED = "DISABLED"
-____exports.EntityStatus.DISABLED_BY_CONTROL_BEHAVIOR = "DISABLED_BY_CONTROL_BEHAVIOR"
-____exports.EntityStatus.DISABLED_BY_SCRIPT = "DISABLED_BY_SCRIPT"
-____exports.EntityStatus.DISCHARGING = "DISCHARGING"
-____exports.EntityStatus.FLUID_INGREDIENT_SHORTAGE = "FLUID_INGREDIENT_SHORTAGE"
-____exports.EntityStatus.FULLY_CHARGED = "FULLY_CHARGED"
-____exports.EntityStatus.FULL_OUTPUT = "FULL_OUTPUT"
-____exports.EntityStatus.ITEM_INGREDIENT_SHORTAGE = "ITEM_INGREDIENT_SHORTAGE"
-____exports.EntityStatus.LAUNCHING_ROCKET = "LAUNCHING_ROCKET"
-____exports.EntityStatus.LOW_INPUT_FLUID = "LOW_INPUT_FLUID"
-____exports.EntityStatus.LOW_POWER = "LOW_POWER"
-____exports.EntityStatus.LOW_TEMPERATURE = "LOW_TEMPERATURE"
-____exports.EntityStatus.MARKED_FOR_DECONSTRUCTION = "MARKED_FOR_DECONSTRUCTION"
-____exports.EntityStatus.MISSING_REQUIRED_FLUID = "MISSING_REQUIRED_FLUID"
-____exports.EntityStatus.MISSING_SCIENCE_PACKS = "MISSING_SCIENCE_PACKS"
-____exports.EntityStatus.NETWORKS_CONNECTED = "NETWORKS_CONNECTED"
-____exports.EntityStatus.NETWORKS_DISCONNECTED = "NETWORKS_DISCONNECTED"
-____exports.EntityStatus.NORMAL = "NORMAL"
-____exports.EntityStatus.NOT_CONNECTED_TO_RAIL = "NOT_CONNECTED_TO_RAIL"
-____exports.EntityStatus.NOT_PLUGGED_IN_ELECTRIC_NETWORK = "NOT_PLUGGED_IN_ELECTRIC_NETWORK"
-____exports.EntityStatus.NO_AMMO = "NO_AMMO"
-____exports.EntityStatus.NO_FUEL = "NO_FUEL"
-____exports.EntityStatus.NO_INGREDIENTS = "NO_INGREDIENTS"
-____exports.EntityStatus.NO_INPUT_FLUID = "NO_INPUT_FLUID"
-____exports.EntityStatus.NO_MINABLE_RESOURCES = "NO_MINABLE_RESOURCES"
-____exports.EntityStatus.NO_MODULES_TO_TRANSMIT = "NO_MODULES_TO_TRANSMIT"
-____exports.EntityStatus.NO_POWER = "NO_POWER"
-____exports.EntityStatus.NO_RECIPE = "NO_RECIPE"
-____exports.EntityStatus.NO_RESEARCH_IN_PROGRESS = "NO_RESEARCH_IN_PROGRESS"
-____exports.EntityStatus.OPENED_BY_CIRCUIT_NETWORK = "OPENED_BY_CIRCUIT_NETWORK"
-____exports.EntityStatus.OUT_OF_LOGISTIC_NETWORK = "OUT_OF_LOGISTIC_NETWORK"
-____exports.EntityStatus.PREPARING_ROCKET_FOR_LAUNCH = "PREPARING_ROCKET_FOR_LAUNCH"
-____exports.EntityStatus.RECHARGING_AFTER_POWER_OUTAGE = "RECHARGING_AFTER_POWER_OUTAGE"
-____exports.EntityStatus.TURNED_OFF_DURING_DAYTIME = "TURNED_OFF_DURING_DAYTIME"
-____exports.EntityStatus.WAITING_FOR_SOURCE_ITEMS = "WAITING_FOR_SOURCE_ITEMS"
-____exports.EntityStatus.WAITING_FOR_SPACE_IN_DESTINATION = "WAITING_FOR_SPACE_IN_DESTINATION"
-____exports.EntityStatus.WAITING_FOR_TARGET_TO_BE_BUILT = "WAITING_FOR_TARGET_TO_BE_BUILT"
-____exports.EntityStatus.WAITING_FOR_TRAIN = "WAITING_FOR_TRAIN"
-____exports.EntityStatus.WAITING_TO_LAUNCH_ROCKET = "WAITING_TO_LAUNCH_ROCKET"
-____exports.EntityStatus.WORKING = "WORKING"
+____exports.ChunkSize = ChunkSize or ({})
+____exports.ChunkSize.CHUNK_512 = "CHUNK_512"
+____exports.ChunkSize.CHUNK_256 = "CHUNK_256"
+____exports.ChunkSize.CHUNK_128 = "CHUNK_128"
+____exports.ChunkSize.CHUNK_064 = "CHUNK_064"
+____exports.ChunkSize.CHUNK_032 = "CHUNK_032"
 ____exports.MinedProduct = {}
 local MinedProduct = ____exports.MinedProduct
 do
@@ -79,4 +35,48 @@ do
     MinedProduct.Type.MinedProductFluid = "kafkatorio.resource.MinedProduct.MinedProductFluid"
     MinedProduct.Type.MinedProductItem = "kafkatorio.resource.MinedProduct.MinedProductItem"
 end
+____exports.FactorioEntityStatus = FactorioEntityStatus or ({})
+____exports.FactorioEntityStatus.WORKING = "WORKING"
+____exports.FactorioEntityStatus.NORMAL = "NORMAL"
+____exports.FactorioEntityStatus.NO_POWER = "NO_POWER"
+____exports.FactorioEntityStatus.LOW_POWER = "LOW_POWER"
+____exports.FactorioEntityStatus.NO_FUEL = "NO_FUEL"
+____exports.FactorioEntityStatus.DISABLED_BY_CONTROL_BEHAVIOR = "DISABLED_BY_CONTROL_BEHAVIOR"
+____exports.FactorioEntityStatus.OPENED_BY_CIRCUIT_NETWORK = "OPENED_BY_CIRCUIT_NETWORK"
+____exports.FactorioEntityStatus.CLOSED_BY_CIRCUIT_NETWORK = "CLOSED_BY_CIRCUIT_NETWORK"
+____exports.FactorioEntityStatus.DISABLED_BY_SCRIPT = "DISABLED_BY_SCRIPT"
+____exports.FactorioEntityStatus.MARKED_FOR_DECONSTRUCTION = "MARKED_FOR_DECONSTRUCTION"
+____exports.FactorioEntityStatus.NOT_PLUGGED_IN_ELECTRIC_NETWORK = "NOT_PLUGGED_IN_ELECTRIC_NETWORK"
+____exports.FactorioEntityStatus.NETWORKS_CONNECTED = "NETWORKS_CONNECTED"
+____exports.FactorioEntityStatus.NETWORKS_DISCONNECTED = "NETWORKS_DISCONNECTED"
+____exports.FactorioEntityStatus.CHARGING = "CHARGING"
+____exports.FactorioEntityStatus.DISCHARGING = "DISCHARGING"
+____exports.FactorioEntityStatus.FULLY_CHARGED = "FULLY_CHARGED"
+____exports.FactorioEntityStatus.OUT_OF_LOGISTIC_NETWORK = "OUT_OF_LOGISTIC_NETWORK"
+____exports.FactorioEntityStatus.NO_RECIPE = "NO_RECIPE"
+____exports.FactorioEntityStatus.NO_INGREDIENTS = "NO_INGREDIENTS"
+____exports.FactorioEntityStatus.NO_INPUT_FLUID = "NO_INPUT_FLUID"
+____exports.FactorioEntityStatus.NO_RESEARCH_IN_PROGRESS = "NO_RESEARCH_IN_PROGRESS"
+____exports.FactorioEntityStatus.NO_MINABLE_RESOURCES = "NO_MINABLE_RESOURCES"
+____exports.FactorioEntityStatus.LOW_INPUT_FLUID = "LOW_INPUT_FLUID"
+____exports.FactorioEntityStatus.FLUID_INGREDIENT_SHORTAGE = "FLUID_INGREDIENT_SHORTAGE"
+____exports.FactorioEntityStatus.FULL_OUTPUT = "FULL_OUTPUT"
+____exports.FactorioEntityStatus.ITEM_INGREDIENT_SHORTAGE = "ITEM_INGREDIENT_SHORTAGE"
+____exports.FactorioEntityStatus.MISSING_REQUIRED_FLUID = "MISSING_REQUIRED_FLUID"
+____exports.FactorioEntityStatus.MISSING_SCIENCE_PACKS = "MISSING_SCIENCE_PACKS"
+____exports.FactorioEntityStatus.WAITING_FOR_SOURCE_ITEMS = "WAITING_FOR_SOURCE_ITEMS"
+____exports.FactorioEntityStatus.WAITING_FOR_SPACE_IN_DESTINATION = "WAITING_FOR_SPACE_IN_DESTINATION"
+____exports.FactorioEntityStatus.PREPARING_ROCKET_FOR_LAUNCH = "PREPARING_ROCKET_FOR_LAUNCH"
+____exports.FactorioEntityStatus.WAITING_TO_LAUNCH_ROCKET = "WAITING_TO_LAUNCH_ROCKET"
+____exports.FactorioEntityStatus.LAUNCHING_ROCKET = "LAUNCHING_ROCKET"
+____exports.FactorioEntityStatus.NO_MODULES_TO_TRANSMIT = "NO_MODULES_TO_TRANSMIT"
+____exports.FactorioEntityStatus.RECHARGING_AFTER_POWER_OUTAGE = "RECHARGING_AFTER_POWER_OUTAGE"
+____exports.FactorioEntityStatus.WAITING_FOR_TARGET_TO_BE_BUILT = "WAITING_FOR_TARGET_TO_BE_BUILT"
+____exports.FactorioEntityStatus.WAITING_FOR_TRAIN = "WAITING_FOR_TRAIN"
+____exports.FactorioEntityStatus.NO_AMMO = "NO_AMMO"
+____exports.FactorioEntityStatus.LOW_TEMPERATURE = "LOW_TEMPERATURE"
+____exports.FactorioEntityStatus.DISABLED = "DISABLED"
+____exports.FactorioEntityStatus.TURNED_OFF_DURING_DAYTIME = "TURNED_OFF_DURING_DAYTIME"
+____exports.FactorioEntityStatus.NOT_CONNECTED_TO_RAIL = "NOT_CONNECTED_TO_RAIL"
+____exports.FactorioEntityStatus.CANT_DIVIDE_SEGMENTS = "CANT_DIVIDE_SEGMENTS"
 return ____exports

modules/events-mod/src/generated/lua/main/commands/kafkatorioCommands.lua

@@ -7,8 +7,8 @@ local ____global_2Dinit = require("main.global-init")
 local initGlobal = ____global_2Dinit.initGlobal
 local ____prototypeUpdates = require("main.events.prototypeUpdates")
 local emitPrototypes = ____prototypeUpdates.emitPrototypes
-local ____queue = require("main.queue.queue")
-local EventDataQueue = ____queue.default
+local ____EventDataQueue = require("main.emitting.EventDataQueue")
+local EventDataQueue = ____EventDataQueue.default
 commands.add_command(
     "kafkatorio",
     "kafkatorio innit bruv",

modules/events-mod/src/generated/lua/main/emitting/KafkatorioPacketQueue.lua

@@ -3,6 +3,10 @@ local __TS__Class = ____lualib.__TS__Class
 local __TS__SetDescriptor = ____lualib.__TS__SetDescriptor
 local __TS__New = ____lualib.__TS__New
 local ____exports = {}
+--- Simple ordered queue for {@link KafkatorioPacketData}.
+-- 
+-- Implemented so printing multiple packets won't flood the output log, causing messages to be
+-- missed or garbled.
 ____exports.KafkatorioPacketQueueManager = __TS__Class()
 local KafkatorioPacketQueueManager = ____exports.KafkatorioPacketQueueManager
 KafkatorioPacketQueueManager.name = "KafkatorioPacketQueueManager"

modules/events-mod/src/generated/lua/main/events/converters.lua

@@ -1,19 +1,12 @@
 local ____exports = {}
 local ____kafkatorio_2Dschema = require("generated.kafkatorio-schema")
-local EntityStatus = ____kafkatorio_2Dschema.EntityStatus
-local FactorioEntityData = ____kafkatorio_2Dschema.FactorioEntityData
+local ChunkSize = ____kafkatorio_2Dschema.ChunkSize
+local FactorioEntityStatus = ____kafkatorio_2Dschema.FactorioEntityStatus
 local MinedProduct = ____kafkatorio_2Dschema.MinedProduct
 ____exports.Converters = {}
 local Converters = ____exports.Converters
 do
-    local convertEntityStatus, EntityStatusRecord
-    function convertEntityStatus(status)
-        if status == nil then
-            return nil
-        else
-            return EntityStatusRecord[status]
-        end
-    end
+    local EntityStatusRecord
     function Converters.prototypeId(____type, name)
         return (____type .. "/") .. name
     end
@@ -42,9 +35,6 @@ do
         end
         return converted
     end
-    function Converters.convertMapTablePosition(position)
-        return {position.x, position.y}
-    end
     function Converters.convertCollisionMaskToNames(cm)
         local masks = {}
         for name in pairs(cm) do
@@ -56,14 +46,14 @@ do
         if entity.type ~= "resource" then
             return nil
         else
-            return {
-                type = FactorioEntityData.Type.Resource,
-                protoId = Converters.prototypeId(entity.type, entity.name),
-                status = convertEntityStatus(entity.status),
-                amount = entity.amount,
-                initialAmount = entity.initial_amount,
-                position = Converters.convertMapTablePosition(entity.position)
-            }
+            return {amount = entity.amount, initialAmount = entity.initial_amount}
+        end
+    end
+    local function convertEntityStatus(status)
+        if status == nil then
+            return nil
+        else
+            return EntityStatusRecord[status]
         end
     end
     function Converters.prototypeIdEntity(entity)
@@ -72,15 +62,23 @@ do
     function Converters.tilePositionToChunkPosition(mapPosition)
         return {
             floor(mapPosition.x / 32),
-            floor(mapPosition.y)
+            floor(mapPosition.y / 32),
+            ChunkSize.CHUNK_032
         }
     end
     function Converters.mapPositionToChunkPosition(mapPosition)
         return {
             floor(mapPosition.x / 32),
-            floor(mapPosition.y)
+            floor(mapPosition.y / 32),
+            ChunkSize.CHUNK_032
         }
     end
+    function Converters.entityPosition(position)
+        return {position.x, position.y}
+    end
+    function Converters.chunkPosition(position)
+        return {position.x, position.y, ChunkSize.CHUNK_032}
+    end
     function Converters.entityStatus(status)
         if status == nil then
             return nil
@@ -116,50 +114,56 @@ do
         data.afkTime = player.afk_time
         data.isConnected = player.connected
     end
+    function Converters.collisionBox(collision_box)
+        return {
+            leftTop = ____exports.Converters.entityPosition(collision_box.left_top),
+            rightBottom = ____exports.Converters.entityPosition(collision_box.right_bottom)
+        }
+    end
     EntityStatusRecord = {
-        [defines.entity_status.cant_divide_segments] = EntityStatus.CANT_DIVIDE_SEGMENTS,
-        [defines.entity_status.charging] = EntityStatus.CHARGING,
-        [defines.entity_status.closed_by_circuit_network] = EntityStatus.CLOSED_BY_CIRCUIT_NETWORK,
-        [defines.entity_status.disabled] = EntityStatus.DISABLED,
-        [defines.entity_status.disabled_by_control_behavior] = EntityStatus.DISABLED_BY_CONTROL_BEHAVIOR,
-        [defines.entity_status.disabled_by_script] = EntityStatus.DISABLED_BY_SCRIPT,
-        [defines.entity_status.discharging] = EntityStatus.DISCHARGING,
-        [defines.entity_status.fluid_ingredient_shortage] = EntityStatus.FLUID_INGREDIENT_SHORTAGE,
-        [defines.entity_status.fully_charged] = EntityStatus.FULLY_CHARGED,
-        [defines.entity_status.full_output] = EntityStatus.FULL_OUTPUT,
-        [defines.entity_status.item_ingredient_shortage] = EntityStatus.ITEM_INGREDIENT_SHORTAGE,
-        [defines.entity_status.launching_rocket] = EntityStatus.LAUNCHING_ROCKET,
-        [defines.entity_status.low_input_fluid] = EntityStatus.LOW_INPUT_FLUID,
-        [defines.entity_status.low_power] = EntityStatus.LOW_POWER,
-        [defines.entity_status.low_temperature] = EntityStatus.LOW_TEMPERATURE,
-        [defines.entity_status.marked_for_deconstruction] = EntityStatus.MARKED_FOR_DECONSTRUCTION,
-        [defines.entity_status.missing_required_fluid] = EntityStatus.MISSING_REQUIRED_FLUID,
-        [defines.entity_status.missing_science_packs] = EntityStatus.MISSING_SCIENCE_PACKS,
-        [defines.entity_status.networks_connected] = EntityStatus.NETWORKS_CONNECTED,
-        [defines.entity_status.networks_disconnected] = EntityStatus.NETWORKS_DISCONNECTED,
-        [defines.entity_status.normal] = EntityStatus.NORMAL,
-        [defines.entity_status.not_connected_to_rail] = EntityStatus.NOT_CONNECTED_TO_RAIL,
-        [defines.entity_status.not_plugged_in_electric_network] = EntityStatus.NOT_PLUGGED_IN_ELECTRIC_NETWORK,
-        [defines.entity_status.no_ammo] = EntityStatus.NO_AMMO,
-        [defines.entity_status.no_fuel] = EntityStatus.NO_FUEL,
-        [defines.entity_status.no_ingredients] = EntityStatus.NO_INGREDIENTS,
-        [defines.entity_status.no_input_fluid] = EntityStatus.NO_INPUT_FLUID,
-        [defines.entity_status.no_minable_resources] = EntityStatus.NO_MINABLE_RESOURCES,
-        [defines.entity_status.no_modules_to_transmit] = EntityStatus.NO_MODULES_TO_TRANSMIT,
-        [defines.entity_status.no_power] = EntityStatus.NO_POWER,
-        [defines.entity_status.no_recipe] = EntityStatus.NO_RECIPE,
-        [defines.entity_status.no_research_in_progress] = EntityStatus.NO_RESEARCH_IN_PROGRESS,
-        [defines.entity_status.opened_by_circuit_network] = EntityStatus.OPENED_BY_CIRCUIT_NETWORK,
-        [defines.entity_status.out_of_logistic_network] = EntityStatus.OUT_OF_LOGISTIC_NETWORK,
-        [defines.entity_status.preparing_rocket_for_launch] = EntityStatus.PREPARING_ROCKET_FOR_LAUNCH,
-        [defines.entity_status.recharging_after_power_outage] = EntityStatus.RECHARGING_AFTER_POWER_OUTAGE,
-        [defines.entity_status.turned_off_during_daytime] = EntityStatus.TURNED_OFF_DURING_DAYTIME,
-        [defines.entity_status.waiting_for_source_items] = EntityStatus.WAITING_FOR_SOURCE_ITEMS,
-        [defines.entity_status.waiting_for_space_in_destination] = EntityStatus.WAITING_FOR_SPACE_IN_DESTINATION,
-        [defines.entity_status.waiting_for_target_to_be_built] = EntityStatus.WAITING_FOR_TARGET_TO_BE_BUILT,
-        [defines.entity_status.waiting_for_train] = EntityStatus.WAITING_FOR_TRAIN,
-        [defines.entity_status.waiting_to_launch_rocket] = EntityStatus.WAITING_TO_LAUNCH_ROCKET,
-        [defines.entity_status.working] = EntityStatus.WORKING
+        [defines.entity_status.cant_divide_segments] = FactorioEntityStatus.CANT_DIVIDE_SEGMENTS,
+        [defines.entity_status.charging] = FactorioEntityStatus.CHARGING,
+        [defines.entity_status.closed_by_circuit_network] = FactorioEntityStatus.CLOSED_BY_CIRCUIT_NETWORK,
+        [defines.entity_status.disabled] = FactorioEntityStatus.DISABLED,
+        [defines.entity_status.disabled_by_control_behavior] = FactorioEntityStatus.DISABLED_BY_CONTROL_BEHAVIOR,
+        [defines.entity_status.disabled_by_script] = FactorioEntityStatus.DISABLED_BY_SCRIPT,
+        [defines.entity_status.discharging] = FactorioEntityStatus.DISCHARGING,
+        [defines.entity_status.fluid_ingredient_shortage] = FactorioEntityStatus.FLUID_INGREDIENT_SHORTAGE,
+        [defines.entity_status.fully_charged] = FactorioEntityStatus.FULLY_CHARGED,
+        [defines.entity_status.full_output] = FactorioEntityStatus.FULL_OUTPUT,
+        [defines.entity_status.item_ingredient_shortage] = FactorioEntityStatus.ITEM_INGREDIENT_SHORTAGE,
+        [defines.entity_status.launching_rocket] = FactorioEntityStatus.LAUNCHING_ROCKET,
+        [defines.entity_status.low_input_fluid] = FactorioEntityStatus.LOW_INPUT_FLUID,
+        [defines.entity_status.low_power] = FactorioEntityStatus.LOW_POWER,
+        [defines.entity_status.low_temperature] = FactorioEntityStatus.LOW_TEMPERATURE,
+        [defines.entity_status.marked_for_deconstruction] = FactorioEntityStatus.MARKED_FOR_DECONSTRUCTION,
+        [defines.entity_status.missing_required_fluid] = FactorioEntityStatus.MISSING_REQUIRED_FLUID,
+        [defines.entity_status.missing_science_packs] = FactorioEntityStatus.MISSING_SCIENCE_PACKS,
+        [defines.entity_status.networks_connected] = FactorioEntityStatus.NETWORKS_CONNECTED,
+        [defines.entity_status.networks_disconnected] = FactorioEntityStatus.NETWORKS_DISCONNECTED,
+        [defines.entity_status.normal] = FactorioEntityStatus.NORMAL,
+        [defines.entity_status.not_connected_to_rail] = FactorioEntityStatus.NOT_CONNECTED_TO_RAIL,
+        [defines.entity_status.not_plugged_in_electric_network] = FactorioEntityStatus.NOT_PLUGGED_IN_ELECTRIC_NETWORK,
+        [defines.entity_status.no_ammo] = FactorioEntityStatus.NO_AMMO,
+        [defines.entity_status.no_fuel] = FactorioEntityStatus.NO_FUEL,
+        [defines.entity_status.no_ingredients] = FactorioEntityStatus.NO_INGREDIENTS,
+        [defines.entity_status.no_input_fluid] = FactorioEntityStatus.NO_INPUT_FLUID,
+        [defines.entity_status.no_minable_resources] = FactorioEntityStatus.NO_MINABLE_RESOURCES,
+        [defines.entity_status.no_modules_to_transmit] = FactorioEntityStatus.NO_MODULES_TO_TRANSMIT,
+        [defines.entity_status.no_power] = FactorioEntityStatus.NO_POWER,
+        [defines.entity_status.no_recipe] = FactorioEntityStatus.NO_RECIPE,
+        [defines.entity_status.no_research_in_progress] = FactorioEntityStatus.NO_RESEARCH_IN_PROGRESS,
+        [defines.entity_status.opened_by_circuit_network] = FactorioEntityStatus.OPENED_BY_CIRCUIT_NETWORK,
+        [defines.entity_status.out_of_logistic_network] = FactorioEntityStatus.OUT_OF_LOGISTIC_NETWORK,
+        [defines.entity_status.preparing_rocket_for_launch] = FactorioEntityStatus.PREPARING_ROCKET_FOR_LAUNCH,
+        [defines.entity_status.recharging_after_power_outage] = FactorioEntityStatus.RECHARGING_AFTER_POWER_OUTAGE,
+        [defines.entity_status.turned_off_during_daytime] = FactorioEntityStatus.TURNED_OFF_DURING_DAYTIME,
+        [defines.entity_status.waiting_for_source_items] = FactorioEntityStatus.WAITING_FOR_SOURCE_ITEMS,
+        [defines.entity_status.waiting_for_space_in_destination] = FactorioEntityStatus.WAITING_FOR_SPACE_IN_DESTINATION,
+        [defines.entity_status.waiting_for_target_to_be_built] = FactorioEntityStatus.WAITING_FOR_TARGET_TO_BE_BUILT,
+        [defines.entity_status.waiting_for_train] = FactorioEntityStatus.WAITING_FOR_TRAIN,
+        [defines.entity_status.waiting_to_launch_rocket] = FactorioEntityStatus.WAITING_TO_LAUNCH_ROCKET,
+        [defines.entity_status.working] = FactorioEntityStatus.WORKING
     }
 end
 return ____exports

modules/events-mod/src/generated/lua/main/events/handlers/EntityUpdateHandler.lua

@@ -3,7 +3,6 @@ local __TS__Class = ____lualib.__TS__Class
 local __TS__New = ____lualib.__TS__New
 local ____exports = {}
 local ____kafkatorio_2Dschema = require("generated.kafkatorio-schema")
-local FactorioEntityData = ____kafkatorio_2Dschema.FactorioEntityData
 local KafkatorioPacketData = ____kafkatorio_2Dschema.KafkatorioPacketData
 local ____converters = require("main.events.converters")
 local Converters = ____converters.Converters
@@ -22,21 +21,26 @@ function EntityUpdatesHandler.prototype.handleMinedTileEvent(self, event)
     local tilesByChunk = {}
     for ____, tile in ipairs(event.tiles) do
         local chunkPosition = Converters.tilePositionToChunkPosition(tile.position)
-        tilesByChunk[chunkPosition] = tile
+        if not (tilesByChunk[chunkPosition] ~= nil) then
+            tilesByChunk[chunkPosition] = {}
+        end
+        local tiles = tilesByChunk[chunkPosition]
+        tiles[#tiles + 1] = tile
+        tilesByChunk[chunkPosition] = tiles
     end
     for chunkPos, tiles in pairs(tilesByChunk) do
         local entities = {}
-        for tile in pairs(tiles) do
-            local resources = ____exports.EntityUpdatesHandler:getTileResourceEntities(surface, chunkPos)
-            for ____, resource in ipairs(resources) do
+        for ____, tile in ipairs(tiles) do
+            local tileEntities = ____exports.EntityUpdatesHandler:getTileResourceEntities(surface, tile.position)
+            for ____, resource in ipairs(tileEntities) do
                 entities[#entities + 1] = resource
             end
         end
-        ____exports.EntityUpdatesHandler:throttleResourceEntitiesUpdate(
+        ____exports.EntityUpdatesHandler:throttleResourcesUpdate(
             Converters.eventNameString(event.name),
             event.tick,
             event.surface_index,
-            {chunkPos[1], chunkPos[2]},
+            chunkPos,
             entities
         )
     end
@@ -49,11 +53,11 @@ function EntityUpdatesHandler.prototype.handleBuiltEntityEvent(self, event)
 end
 function EntityUpdatesHandler.prototype.handleChunkGeneratedEvent(self, event)
     local entities = ____exports.EntityUpdatesHandler:getAreaResourceEntities(event.surface, event.area)
-    ____exports.EntityUpdatesHandler:throttleResourceEntitiesUpdate(
+    ____exports.EntityUpdatesHandler:throttleResourcesUpdate(
         Converters.eventNameString(event.name),
         event.tick,
         event.surface.index,
-        {event.position.x, event.position.y},
+        Converters.chunkPosition(event.position),
         entities
     )
 end
@@ -67,30 +71,24 @@ function EntityUpdatesHandler.throttleEntityUpdate(self, event, entity)
         surfaceIndex = entity.surface.index,
         chunkPosition = Converters.mapPositionToChunkPosition(entity.position)
     }
-    local ____FactorioEntityData_Type_Standard_2 = FactorioEntityData.Type.Standard
-    local ____Converters_prototypeIdEntity_result_3 = Converters.prototypeIdEntity(entity)
-    local ____entity_graphics_variation_4 = entity.graphics_variation
-    local ____entity_health_5 = entity.health
-    local ____entity_active_6 = entity.active
-    local ____entity_rotatable_7 = entity.rotatable
+    local ____entity_unit_number_2 = entity.unit_number
+    local ____entity_graphics_variation_3 = entity.graphics_variation
+    local ____entity_health_4 = entity.health
+    local ____entity_active_5 = entity.active
+    local ____entity_rotatable_6 = entity.rotatable
     local ____entity_last_user_index_0 = entity.last_user
     if ____entity_last_user_index_0 ~= nil then
         ____entity_last_user_index_0 = ____entity_last_user_index_0.index
     end
     local entityUpdate = {
-        type = ____FactorioEntityData_Type_Standard_2,
-        protoId = ____Converters_prototypeIdEntity_result_3,
-        graphicsVariation = ____entity_graphics_variation_4,
-        health = ____entity_health_5,
-        isActive = ____entity_active_6,
-        isRotatable = ____entity_rotatable_7,
+        unitNumber = ____entity_unit_number_2,
+        graphicsVariation = ____entity_graphics_variation_3,
+        health = ____entity_health_4,
+        isActive = ____entity_active_5,
+        isRotatable = ____entity_rotatable_6,
         lastUser = ____entity_last_user_index_0,
-        localisedDescription = nil,
-        localisedName = nil,
-        position = {entity.position.x, entity.position.y},
         status = Converters.entityStatus(entity.status)
     }
-    local entityKey = tostring(entity.unit_number)
     EventDataCache:throttle(
         entityUpdateKey,
         KafkatorioPacketData.Type.MapChunkEntityUpdate,
@@ -98,77 +96,73 @@ function EntityUpdatesHandler.throttleEntityUpdate(self, event, entity)
             if data.events == nil then
                 data.events = {}
             end
-            local ____data_events_8, ____eventName_9 = data.events, eventName
-            if ____data_events_8[____eventName_9] == nil then
-                ____data_events_8[____eventName_9] = {}
+            local ____data_events_7, ____eventName_8 = data.events, eventName
+            if ____data_events_7[____eventName_8] == nil then
+                ____data_events_7[____eventName_8] = {}
             end
-            local ____data_events_eventName_10 = data.events[eventName]
-            ____data_events_eventName_10[#____data_events_eventName_10 + 1] = event.tick
-            if data.distinctEntities == nil then
-                data.distinctEntities = {}
+            local ____data_events_eventName_9 = data.events[eventName]
+            ____data_events_eventName_9[#____data_events_eventName_9 + 1] = event.tick
+            if data.entitiesXY == nil then
+                data.entitiesXY = {}
+            end
+            local ____data_entitiesXY_10, ____tostring_result_11 = data.entitiesXY, tostring(entity.position.x)
+            if ____data_entitiesXY_10[____tostring_result_11] == nil then
+                ____data_entitiesXY_10[____tostring_result_11] = {}
+            end
+            local ____data_entitiesXY_tostring_result_12, ____tostring_result_13 = data.entitiesXY[tostring(entity.position.x)], tostring(entity.position.y)
+            if ____data_entitiesXY_tostring_result_12[____tostring_result_13] == nil then
+                ____data_entitiesXY_tostring_result_12[____tostring_result_13] = entityUpdate
             end
-            data.distinctEntities[entityKey] = entityUpdate
         end
     )
 end
-function EntityUpdatesHandler.throttleResourceEntitiesUpdate(self, eventName, eventTick, surfaceIndex, chunkPosition, entities)
+function EntityUpdatesHandler.throttleResourcesUpdate(self, eventName, eventTick, surfaceIndex, chunkPosition, entities)
     local entitiesByProtoId = {}
     for ____, entity in ipairs(entities) do
-        local ____entitiesByProtoId_11, ____entity_protoId_12 = entitiesByProtoId, entity.protoId
-        if ____entitiesByProtoId_11[____entity_protoId_12] == nil then
-            ____entitiesByProtoId_11[____entity_protoId_12] = {}
+        local protoId = Converters.prototypeIdEntity(entity)
+        if entitiesByProtoId[protoId] == nil then
+            entitiesByProtoId[protoId] = {}
         end
-        local ____entitiesByProtoId_entity_protoId_13 = entitiesByProtoId[entity.protoId]
-        ____entitiesByProtoId_entity_protoId_13[#____entitiesByProtoId_entity_protoId_13 + 1] = entity
+        local ____entitiesByProtoId_protoId_14 = entitiesByProtoId[protoId]
+        ____entitiesByProtoId_protoId_14[#____entitiesByProtoId_protoId_14 + 1] = entity
     end
     for protoId, resourceEntities in pairs(entitiesByProtoId) do
-        local entityUpdateKey = {protoId = protoId, surfaceIndex = surfaceIndex, chunkPosition = {chunkPosition[1], chunkPosition[2]}}
+        local entityUpdateKey = {protoId = protoId, surfaceIndex = surfaceIndex, chunkPosition = chunkPosition}
         EventDataCache:throttle(
             entityUpdateKey,
-            KafkatorioPacketData.Type.MapChunkEntityUpdate,
+            KafkatorioPacketData.Type.MapChunkResourceUpdate,
             function(data)
                 if data.events == nil then
                     data.events = {}
                 end
-                local ____data_events_14, ____eventName_15 = data.events, eventName
-                if ____data_events_14[____eventName_15] == nil then
-                    ____data_events_14[____eventName_15] = {}
+                local ____data_events_15, ____eventName_16 = data.events, eventName
+                if ____data_events_15[____eventName_16] == nil then
+                    ____data_events_15[____eventName_16] = {}
                 end
-                local ____data_events_eventName_16 = data.events[eventName]
-                ____data_events_eventName_16[#____data_events_eventName_16 + 1] = eventTick
+                local ____data_events_eventName_17 = data.events[eventName]
+                ____data_events_eventName_17[#____data_events_eventName_17 + 1] = eventTick
                 for ____, entity in ipairs(resourceEntities) do
-                    local x, y = table.unpack(entity.position)
-                    local entityKey = (tostring(x) .. ",") .. tostring(y)
-                    if data.distinctEntities == nil then
-                        data.distinctEntities = {}
+                    local resourceUpdate = Converters.convertResourceEntity(entity)
+                    if resourceUpdate ~= nil then
+                        if data.resourcesXY == nil then
+                            data.resourcesXY = {}
+                        end
+                        local ____data_resourcesXY_18, ____tostring_result_19 = data.resourcesXY, tostring(entity.position.x)
+                        if ____data_resourcesXY_18[____tostring_result_19] == nil then
+                            ____data_resourcesXY_18[____tostring_result_19] = {}
+                        end
+                        data.resourcesXY[tostring(entity.position.x)][tostring(entity.position.y)] = resourceUpdate
                     end
-                    data.distinctEntities[entityKey] = entity
                 end
             end
         )
     end
 end
 function EntityUpdatesHandler.getTileResourceEntities(self, surface, mapPosition)
-    local rawEntities = surface.find_entities_filtered({position = mapPosition, collision_mask = "resource-layer"})
-    local resources = {}
-    for ____, rawEntity in ipairs(rawEntities) do
-        local entity = Converters.convertResourceEntity(rawEntity)
-        if entity ~= nil then
-            resources[#resources + 1] = entity
-        end
-    end
-    return resources
+    return surface.find_entities_filtered({position = mapPosition, collision_mask = "resource-layer"})
 end
 function EntityUpdatesHandler.getAreaResourceEntities(self, surface, area)
-    local rawEntities = surface.find_entities_filtered({area = area, collision_mask = "resource-layer"})
-    local resources = {}
-    for ____, rawEntity in ipairs(rawEntities) do
-        local entity = Converters.convertResourceEntity(rawEntity)
-        if entity ~= nil then
-            resources[#resources + 1] = entity
-        end
-    end
-    return resources
+    return surface.find_entities_filtered({area = area, collision_mask = "resource-layer"})
 end
 local EntityUpdates = __TS__New(____exports.EntityUpdatesHandler)
 ____exports.default = EntityUpdates

modules/events-mod/src/generated/lua/main/events/handlers/MapChunkUpdateHandler.lua

@@ -20,7 +20,7 @@ function MapChunkUpdateHandler.prototype.handleChunkGeneratedEvent(self, event,
     local tiles = event.surface.find_tiles_filtered({area = event.area})
     MapChunkUpdateHandler:mapTilesUpdateDebounce(
         event.surface,
-        Converters.convertMapTablePosition(event.position),
+        Converters.chunkPosition(event.position),
         tiles,
         event,
         nil,
@@ -123,7 +123,10 @@ function MapChunkUpdateHandler.prototype.handlePreChunkDeleted(self, event)
         return
     end
     for ____, position in ipairs(event.positions) do
-        local key = {surfaceIndex = surface.index, chunkPosition = {position.x, position.y}}
+        local key = {
+            surfaceIndex = surface.index,
+            chunkPosition = Converters.chunkPosition(position)
+        }
         EventUpdates:debounce(
             key,
             KafkatorioPacketData.Type.MapChunkTileUpdate,
@@ -187,10 +190,7 @@ end
 function MapChunkUpdateHandler.groupTiles(self, tiles)
     local mapChunkPositionToTiles = __TS__New(Map)
     for ____, tile in ipairs(tiles) do
-        local chunkPosition = {
-            math.floor(tile.position.x / 32),
-            math.floor(tile.position.y / 32)
-        }
+        local chunkPosition = Converters.tilePositionToChunkPosition(tile.position)
         if not mapChunkPositionToTiles:has(chunkPosition) then
             mapChunkPositionToTiles:set(chunkPosition, {})
         end

modules/events-mod/src/generated/lua/main/events/prototypeUpdates.lua

@@ -30,7 +30,7 @@ end
 function getEntityPrototypes()
     local protos = {}
     for ____, entity in pairs(game.entity_prototypes) do
-        local key = (entity.group.name .. "/") .. entity.subgroup.name
+        local key = (((entity.group.name .. "/") .. entity.subgroup.name) .. "/") .. entity.type
         if protos[key] == nil then
             protos[key] = {}
         end
@@ -42,8 +42,10 @@ function getEntityPrototypes()
             isBuilding = entity.is_building,
             isEntityWithOwner = entity.is_entity_with_owner,
             isMilitaryTarget = entity.is_military_target,
-            maxHealth = entity.max_health
+            maxHealth = entity.max_health,
+            collisionBox = Converters.collisionBox(entity.collision_box)
         }
+        local ____ = entity.infinite_resource
         if entity.map_color ~= nil then
             entityProto.mapColour = Converters.mapColour(entity.map_color)
         end

modules/events-mod/src/generated/lua/main/events/registerEventListeners.lua

@@ -9,12 +9,12 @@ local ____EventDataCache = require("main.emitting.EventDataCache")
 local EventCacheService = ____EventDataCache.default
 local ____MapChunkUpdateHandler = require("main.events.handlers.MapChunkUpdateHandler")
 local MapChunkUpdate = ____MapChunkUpdateHandler.default
-local ____queue = require("main.queue.queue")
-local EventDataQueue = ____queue.default
 local ____KafkatorioPacketQueue = require("main.emitting.KafkatorioPacketQueue")
 local KafkatorioPacketQueue = ____KafkatorioPacketQueue.default
 local ____PacketEmitter = require("main.emitting.PacketEmitter")
 local PacketEmitter = ____PacketEmitter.default
+local ____EventDataQueue = require("main.emitting.EventDataQueue")
+local EventDataQueue = ____EventDataQueue.default
 script.on_event(
     defines.events.on_tick,
     function(event)

modules/events-mod/src/generated/lua/main/global-init.lua

@@ -1,8 +1,8 @@
 local ____exports = {}
 local ____EventDataCache = require("main.emitting.EventDataCache")
 local EventUpdates = ____EventDataCache.default
-local ____queue = require("main.queue.queue")
-local EventDataQueueManager = ____queue.EventDataQueueManager
+local ____EventDataQueue = require("main.emitting.EventDataQueue")
+local EventDataQueueManager = ____EventDataQueue.EventDataQueueManager
 function ____exports.initGlobal(force)
     if force == nil then
         force = false

modules/events-mod/src/generated/lua/main/settings/KafkatorioSettings.lua

@@ -34,7 +34,13 @@ end
 function KafkatorioSettingsConfig.cacheDurationSettingName(self, ____type)
     return "kafkatorio:cache-expiration-seconds_" .. __TS__StringReplaceAll(____type, ".", "-")
 end
-KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds = {["kafkatorio.packet.keyed.PlayerUpdate"] = 0.5, ["kafkatorio.packet.keyed.MapChunkTileUpdate"] = 30, ["kafkatorio.packet.keyed.MapChunkEntityUpdate"] = 30, ["kafkatorio.packet.keyed.EntityUpdate"] = 5}
+KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds = {
+    ["kafkatorio.packet.keyed.PlayerUpdate"] = 0.5,
+    ["kafkatorio.packet.keyed.MapChunkTileUpdate"] = 30,
+    ["kafkatorio.packet.keyed.MapChunkEntityUpdate"] = 30,
+    ["kafkatorio.packet.keyed.MapChunkResourceUpdate"] = 30,
+    ["kafkatorio.packet.keyed.EntityUpdate"] = 5
+}
 local KafkatorioSettings = __TS__New(____exports.KafkatorioSettingsConfig)
 ____exports.default = KafkatorioSettings
 return ____exports

modules/events-mod/src/main/resources/mod-data/locale/en/locale.cfg

@@ -2,15 +2,17 @@
 
 kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-PlayerUpdate=Player update rate (seconds)
 kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkTileUpdate=Map tile update rate (seconds)
-kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkEntityUpdate=Entity update rate (seconds)
+kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkEntityUpdate=Map entity update rate (seconds)
+kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkResourceUpdate=Map resource update rate (seconds)
 kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-EntityUpdate=Entity update rate (seconds)
 
 
 [mod-setting-description]
 
 kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-PlayerUpdate=How frequently Player updates are sent to the Kafkatorio server, in seconds. Lower means updates are sent more frequently, which may cause network congestion or reduce performance. The default (0.25 seconds) is probably fine!
-kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkTileUpdate=How frequently Map updates are sent to the Kafkatorio server, in seconds. Lower means updates are sent more frequently, which may cause network congestion or reduce performance. The default (30 seconds) is probably fine!
-kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkEntityUpdate=How frequently entity updates are sent to the Kafkatorio server, in seconds. Lower means updates are sent more frequently, which may cause network congestion or reduce performance. The default (30 seconds) is probably fine!
+kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkTileUpdate=How frequently map-tile updates are sent to the Kafkatorio server, in seconds. Lower means updates are sent more frequently, which may cause network congestion or reduce performance. The default (30 seconds) is probably fine!
+kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkEntityUpdate=How frequently map-entity updates are sent to the Kafkatorio server, in seconds. Lower means updates are sent more frequently, which may cause network congestion or reduce performance. The default (30 seconds) is probably fine!
+kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-MapChunkResourceUpdate=How frequently map-resource updates are sent to the Kafkatorio server, in seconds. Lower means updates are sent more frequently, which may cause network congestion or reduce performance. The default (30 seconds) is probably fine!
 kafkatorio:cache-expiration-seconds_kafkatorio-packet-keyed-EntityUpdate=How frequently Entity updates are sent to the Kafkatorio server, in seconds. Lower means updates are sent more frequently, which may cause network congestion or reduce performance. The default (5 seconds) is probably fine!
 
 

modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts

@@ -15,6 +15,7 @@ export type KafkatorioPacketData =
   | KafkatorioPacketData.EntityUpdate
   | KafkatorioPacketData.Error
   | KafkatorioPacketData.MapChunkEntityUpdate
+  | KafkatorioPacketData.MapChunkResourceUpdate
   | KafkatorioPacketData.MapChunkTileUpdate
   | KafkatorioPacketData.PlayerUpdate
   | KafkatorioPacketData.PrototypesUpdate
@@ -29,6 +30,7 @@ export namespace KafkatorioPacketData {
     SurfaceUpdate = "kafkatorio.packet.instant.SurfaceUpdate",
     EntityUpdate = "kafkatorio.packet.keyed.EntityUpdate",
     MapChunkEntityUpdate = "kafkatorio.packet.keyed.MapChunkEntityUpdate",
+    MapChunkResourceUpdate = "kafkatorio.packet.keyed.MapChunkResourceUpdate",
     MapChunkTileUpdate = "kafkatorio.packet.keyed.MapChunkTileUpdate",
     PlayerUpdate = "kafkatorio.packet.keyed.PlayerUpdate",
     Error = "kafkatorio.packet.KafkatorioPacketData.Error",
@@ -74,8 +76,8 @@ export namespace KafkatorioPacketData {
     chunkPosition?: MapEntityPosition | null;
     graphicsVariation?: UByte | null;
     health?: Float | null;
-    isActive?: boolean | null;
-    isRotatable?: boolean | null;
+    isActive?: boolean;
+    isRotatable?: boolean;
     lastUser?: UInt | null;
     localisedDescription?: string | null;
     localisedName?: string | null;
@@ -85,7 +87,14 @@ export namespace KafkatorioPacketData {
     type: KafkatorioPacketData.Type.MapChunkEntityUpdate;
     key: MapChunkEntityUpdateKey;
     events?: { [key: EventName]: Tick[] } | null;
-    distinctEntities?: { [key: string]: FactorioEntityData } | null;
+    entitiesXY: FactorioEntityUpdateEntityDictionary;
+  }
+  
+  export interface MapChunkResourceUpdate {
+    type: KafkatorioPacketData.Type.MapChunkResourceUpdate;
+    key: MapChunkResourceUpdateKey;
+    events?: { [key: EventName]: Tick[] } | null;
+    resourcesXY: FactorioEntityUpdateResourceDictionary;
   }
   
   export interface MapChunkTileUpdate {
@@ -96,7 +105,7 @@ export namespace KafkatorioPacketData {
     robot?: EntityIdentifiersData | null;
     force?: ForceIndex | null;
     tileDictionary?: MapTileDictionary | null;
-    isDeleted?: boolean | null;
+    isDeleted?: boolean;
   }
   
   export interface PlayerUpdate {
@@ -110,10 +119,10 @@ export namespace KafkatorioPacketData {
     afkTime?: Tick | null;
     ticksToRespawn?: Tick | null;
     forceIndex?: ForceIndex | null;
-    isAdmin?: boolean | null;
-    isConnected?: boolean | null;
-    isShowOnMap?: boolean | null;
-    isSpectator?: boolean | null;
+    isAdmin?: boolean;
+    isConnected?: boolean;
+    isShowOnMap?: boolean;
+    isSpectator?: boolean;
     lastOnline?: Tick | null;
     onlineTime?: Tick | null;
     position?: MapEntityPosition | null;
@@ -162,6 +171,16 @@ export interface MapChunkEntityUpdateKey {
   chunkPosition: MapChunkPosition;
 }
 
+export type FactorioEntityUpdateEntityDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateEntity } };
+
+export interface MapChunkResourceUpdateKey {
+  protoId: PrototypeId;
+  surfaceIndex: SurfaceIndex;
+  chunkPosition: MapChunkPosition;
+}
+
+export type FactorioEntityUpdateResourceDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateResource } };
+
 export interface MapChunkTileUpdateKey {
   chunkPosition: MapChunkPosition;
   surfaceIndex: SurfaceIndex;
@@ -218,10 +237,11 @@ export namespace FactorioPrototype {
     mapColourFriend?: Colour | null;
     mapColourEnemy?: Colour | null;
     maxHealth: Float;
-    isBuilding: boolean;
-    isEntityWithOwner: boolean;
-    isMilitaryTarget: boolean;
+    isBuilding?: boolean;
+    isEntityWithOwner?: boolean;
+    isMilitaryTarget?: boolean;
     miningProperties?: EntityMiningProperties | null;
+    collisionBox?: MapBoundingBox | null;
   }
   
   export interface MapTile {
@@ -229,7 +249,7 @@ export namespace FactorioPrototype {
     protoId: PrototypeId;
     mapColour: Colour;
     layer: UInt;
-    collisionMasks: List;
+    collisionMasks: string[];
     order: string;
     canBeMined: boolean;
   }
@@ -244,42 +264,9 @@ export type Byte = int8;
 export type MapChunkPosition = [
   x: Int,
   y: Int,
+  chunkSize: ChunkSize,
 ];
 
-export type FactorioEntityData =
-  | FactorioEntityData.Resource
-  | FactorioEntityData.Standard;
-
-export namespace FactorioEntityData {
-  export enum Type {
-    Resource = "kafkatorio.entity.FactorioEntityData.Resource",
-    Standard = "kafkatorio.entity.FactorioEntityData.Standard",
-  }
-  
-  export interface Resource {
-    type: FactorioEntityData.Type.Resource;
-    protoId: PrototypeId;
-    status?: EntityStatus | null;
-    position: MapEntityPosition;
-    amount: UInt;
-    initialAmount?: UInt | null;
-  }
-  
-  export interface Standard {
-    type: FactorioEntityData.Type.Standard;
-    protoId: PrototypeId;
-    status?: EntityStatus | null;
-    position: MapEntityPosition;
-    graphicsVariation?: UByte | null;
-    health?: Float | null;
-    isActive?: boolean | null;
-    isRotatable?: boolean | null;
-    lastUser?: UInt | null;
-    localisedDescription?: string | null;
-    localisedName?: string | null;
-  }
-}
-
 export interface EntityItemGroup {
   name: string;
   type: string;
@@ -291,56 +278,36 @@ export interface EntityMiningProperties {
   products: MinedProduct[] | null;
 }
 
-export type List = any & { __List__: void };
+export interface MapBoundingBox {
+  leftTop: MapEntityPosition;
+  rightBottom: MapEntityPosition;
+}
 
-export enum EntityStatus {
-  CANT_DIVIDE_SEGMENTS = "CANT_DIVIDE_SEGMENTS",
-  CHARGING = "CHARGING",
-  CLOSED_BY_CIRCUIT_NETWORK = "CLOSED_BY_CIRCUIT_NETWORK",
-  DISABLED = "DISABLED",
-  DISABLED_BY_CONTROL_BEHAVIOR = "DISABLED_BY_CONTROL_BEHAVIOR",
-  DISABLED_BY_SCRIPT = "DISABLED_BY_SCRIPT",
-  DISCHARGING = "DISCHARGING",
-  FLUID_INGREDIENT_SHORTAGE = "FLUID_INGREDIENT_SHORTAGE",
-  FULLY_CHARGED = "FULLY_CHARGED",
-  FULL_OUTPUT = "FULL_OUTPUT",
-  ITEM_INGREDIENT_SHORTAGE = "ITEM_INGREDIENT_SHORTAGE",
-  LAUNCHING_ROCKET = "LAUNCHING_ROCKET",
-  LOW_INPUT_FLUID = "LOW_INPUT_FLUID",
-  LOW_POWER = "LOW_POWER",
-  LOW_TEMPERATURE = "LOW_TEMPERATURE",
-  MARKED_FOR_DECONSTRUCTION = "MARKED_FOR_DECONSTRUCTION",
-  MISSING_REQUIRED_FLUID = "MISSING_REQUIRED_FLUID",
-  MISSING_SCIENCE_PACKS = "MISSING_SCIENCE_PACKS",
-  NETWORKS_CONNECTED = "NETWORKS_CONNECTED",
-  NETWORKS_DISCONNECTED = "NETWORKS_DISCONNECTED",
-  NORMAL = "NORMAL",
-  NOT_CONNECTED_TO_RAIL = "NOT_CONNECTED_TO_RAIL",
-  NOT_PLUGGED_IN_ELECTRIC_NETWORK = "NOT_PLUGGED_IN_ELECTRIC_NETWORK",
-  NO_AMMO = "NO_AMMO",
-  NO_FUEL = "NO_FUEL",
-  NO_INGREDIENTS = "NO_INGREDIENTS",
-  NO_INPUT_FLUID = "NO_INPUT_FLUID",
-  NO_MINABLE_RESOURCES = "NO_MINABLE_RESOURCES",
-  NO_MODULES_TO_TRANSMIT = "NO_MODULES_TO_TRANSMIT",
-  NO_POWER = "NO_POWER",
-  NO_RECIPE = "NO_RECIPE",
-  NO_RESEARCH_IN_PROGRESS = "NO_RESEARCH_IN_PROGRESS",
-  OPENED_BY_CIRCUIT_NETWORK = "OPENED_BY_CIRCUIT_NETWORK",
-  OUT_OF_LOGISTIC_NETWORK = "OUT_OF_LOGISTIC_NETWORK",
-  PREPARING_ROCKET_FOR_LAUNCH = "PREPARING_ROCKET_FOR_LAUNCH",
-  RECHARGING_AFTER_POWER_OUTAGE = "RECHARGING_AFTER_POWER_OUTAGE",
-  TURNED_OFF_DURING_DAYTIME = "TURNED_OFF_DURING_DAYTIME",
-  WAITING_FOR_SOURCE_ITEMS = "WAITING_FOR_SOURCE_ITEMS",
-  WAITING_FOR_SPACE_IN_DESTINATION = "WAITING_FOR_SPACE_IN_DESTINATION",
-  WAITING_FOR_TARGET_TO_BE_BUILT = "WAITING_FOR_TARGET_TO_BE_BUILT",
-  WAITING_FOR_TRAIN = "WAITING_FOR_TRAIN",
-  WAITING_TO_LAUNCH_ROCKET = "WAITING_TO_LAUNCH_ROCKET",
-  WORKING = "WORKING",
+export enum ChunkSize {
+  CHUNK_512 = "CHUNK_512",
+  CHUNK_256 = "CHUNK_256",
+  CHUNK_128 = "CHUNK_128",
+  CHUNK_064 = "CHUNK_064",
+  CHUNK_032 = "CHUNK_032",
 }
 
 export type PrototypeKey = Int & { __PrototypeKey__: void };
 
+export interface FactorioEntityUpdateEntity {
+  status?: FactorioEntityStatus | null;
+  unitNumber?: UnitNumber | null;
+  graphicsVariation?: UByte | null;
+  health?: Float | null;
+  isActive?: boolean | null;
+  isRotatable?: boolean | null;
+  lastUser?: UInt | null;
+}
+
+export interface FactorioEntityUpdateResource {
+  amount: UInt;
+  initialAmount?: UInt | null;
+}
+
 export type MinedProduct =
   | MinedProduct.MinedProductFluid
   | MinedProduct.MinedProductItem;
@@ -363,3 +330,49 @@ export namespace MinedProduct {
     resultProtoId: PrototypeId;
   }
 }
+
+export enum FactorioEntityStatus {
+  WORKING = "WORKING",
+  NORMAL = "NORMAL",
+  NO_POWER = "NO_POWER",
+  LOW_POWER = "LOW_POWER",
+  NO_FUEL = "NO_FUEL",
+  DISABLED_BY_CONTROL_BEHAVIOR = "DISABLED_BY_CONTROL_BEHAVIOR",
+  OPENED_BY_CIRCUIT_NETWORK = "OPENED_BY_CIRCUIT_NETWORK",
+  CLOSED_BY_CIRCUIT_NETWORK = "CLOSED_BY_CIRCUIT_NETWORK",
+  DISABLED_BY_SCRIPT = "DISABLED_BY_SCRIPT",
+  MARKED_FOR_DECONSTRUCTION = "MARKED_FOR_DECONSTRUCTION",
+  NOT_PLUGGED_IN_ELECTRIC_NETWORK = "NOT_PLUGGED_IN_ELECTRIC_NETWORK",
+  NETWORKS_CONNECTED = "NETWORKS_CONNECTED",
+  NETWORKS_DISCONNECTED = "NETWORKS_DISCONNECTED",
+  CHARGING = "CHARGING",
+  DISCHARGING = "DISCHARGING",
+  FULLY_CHARGED = "FULLY_CHARGED",
+  OUT_OF_LOGISTIC_NETWORK = "OUT_OF_LOGISTIC_NETWORK",
+  NO_RECIPE = "NO_RECIPE",
+  NO_INGREDIENTS = "NO_INGREDIENTS",
+  NO_INPUT_FLUID = "NO_INPUT_FLUID",
+  NO_RESEARCH_IN_PROGRESS = "NO_RESEARCH_IN_PROGRESS",
+  NO_MINABLE_RESOURCES = "NO_MINABLE_RESOURCES",
+  LOW_INPUT_FLUID = "LOW_INPUT_FLUID",
+  FLUID_INGREDIENT_SHORTAGE = "FLUID_INGREDIENT_SHORTAGE",
+  FULL_OUTPUT = "FULL_OUTPUT",
+  ITEM_INGREDIENT_SHORTAGE = "ITEM_INGREDIENT_SHORTAGE",
+  MISSING_REQUIRED_FLUID = "MISSING_REQUIRED_FLUID",
+  MISSING_SCIENCE_PACKS = "MISSING_SCIENCE_PACKS",
+  WAITING_FOR_SOURCE_ITEMS = "WAITING_FOR_SOURCE_ITEMS",
+  WAITING_FOR_SPACE_IN_DESTINATION = "WAITING_FOR_SPACE_IN_DESTINATION",
+  PREPARING_ROCKET_FOR_LAUNCH = "PREPARING_ROCKET_FOR_LAUNCH",
+  WAITING_TO_LAUNCH_ROCKET = "WAITING_TO_LAUNCH_ROCKET",
+  LAUNCHING_ROCKET = "LAUNCHING_ROCKET",
+  NO_MODULES_TO_TRANSMIT = "NO_MODULES_TO_TRANSMIT",
+  RECHARGING_AFTER_POWER_OUTAGE = "RECHARGING_AFTER_POWER_OUTAGE",
+  WAITING_FOR_TARGET_TO_BE_BUILT = "WAITING_FOR_TARGET_TO_BE_BUILT",
+  WAITING_FOR_TRAIN = "WAITING_FOR_TRAIN",
+  NO_AMMO = "NO_AMMO",
+  LOW_TEMPERATURE = "LOW_TEMPERATURE",
+  DISABLED = "DISABLED",
+  TURNED_OFF_DURING_DAYTIME = "TURNED_OFF_DURING_DAYTIME",
+  NOT_CONNECTED_TO_RAIL = "NOT_CONNECTED_TO_RAIL",
+  CANT_DIVIDE_SEGMENTS = "CANT_DIVIDE_SEGMENTS",
+}

modules/events-mod/src/main/typescript/main/commands/kafkatorioCommands.ts

@@ -1,6 +1,6 @@
 import {initGlobal} from "../global-init";
 import {emitPrototypes} from "../events/prototypeUpdates";
-import EventDataQueue from "../queue/queue";
+import EventDataQueue from "../emitting/EventDataQueue";
 
 
 commands.add_command(

modules/events-mod/src/main/typescript/main/emitting/KafkatorioPacketQueue.ts

@@ -1,5 +1,12 @@
 import {KafkatorioPacketData} from "../../generated/kafkatorio-schema";
 
+
+/**
+ * Simple ordered queue for {@link KafkatorioPacketData}.
+ *
+ * Implemented so printing multiple packets won't flood the output log, causing messages to be
+ * missed or garbled.
+ */
 export class KafkatorioPacketQueueManager {
 
   private size: int = 0

modules/events-mod/src/main/typescript/main/events/converters.ts

@@ -1,16 +1,20 @@
 import {DefinedEventName} from "../types";
 import {
+  ChunkSize,
   Colour,
   EntityMiningProperties,
-  EntityStatus,
   EventName,
-  FactorioEntityData,
+  FactorioEntityStatus,
+  FactorioEntityUpdateResource,
   KafkatorioPacketData,
+  MapBoundingBox,
   MapChunkPosition,
+  MapEntityPosition,
   MinedProduct,
   PrototypeId,
 } from "../../generated/kafkatorio-schema";
 
+
 export namespace Converters {
 
 
@@ -63,13 +67,6 @@ export namespace Converters {
   }
 
 
-  export function convertMapTablePosition(
-      position: MapPositionTable,
-  ): MapChunkPosition {
-    return [position.x, position.y]
-  }
-
-
   export function convertCollisionMaskToNames(cm: CollisionMask): string[] {
     let masks: string[] = []
     for (const [name] of pairs(cm)) {
@@ -81,17 +78,13 @@ export namespace Converters {
 
   export function convertResourceEntity(
       entity: LuaEntity
-  ): FactorioEntityData.Resource | null {
+  ): FactorioEntityUpdateResource | null {
     if (entity.type != "resource") {
       return null
     } else {
       return {
-        type: FactorioEntityData.Type.Resource,
-        protoId: prototypeId(entity.type, entity.name),
-        status: convertEntityStatus(entity.status),
         amount: entity.amount,
         initialAmount: entity.initial_amount,
-        position: convertMapTablePosition(entity.position),
       }
     }
   }
@@ -99,7 +92,7 @@ export namespace Converters {
 
   function convertEntityStatus(
       status: defines.entity_status | undefined
-  ): EntityStatus | null {
+  ): FactorioEntityStatus | null {
     if (status == undefined) {
       return null
     } else {
@@ -119,15 +112,26 @@ export namespace Converters {
 
 
   export function tilePositionToChunkPosition(mapPosition: TilePositionTable): MapChunkPosition {
-    return [floor(mapPosition.x / 32), floor(mapPosition.y)]
+    return [floor(mapPosition.x / 32), floor(mapPosition.y / 32), ChunkSize.CHUNK_032]
   }
 
+
   export function mapPositionToChunkPosition(mapPosition: MapPositionTable): MapChunkPosition {
-    return [floor(mapPosition.x / 32), floor(mapPosition.y)]
+    return [floor(mapPosition.x / 32), floor(mapPosition.y / 32), ChunkSize.CHUNK_032]
   }
 
 
-  export function entityStatus(status: defines.entity_status | undefined): EntityStatus | undefined {
+  export function entityPosition(position: MapPositionTable): MapEntityPosition {
+    return [position.x, position.y];
+  }
+
+
+  export function chunkPosition(position: ChunkPositionTable): MapChunkPosition {
+    return [position.x, position.y, ChunkSize.CHUNK_032]
+  }
+
+
+  export function entityStatus(status: defines.entity_status | undefined): FactorioEntityStatus | undefined {
     if (status == undefined) {
       return undefined
     } else {
@@ -176,51 +180,59 @@ export namespace Converters {
   }
 
 
-  const EntityStatusRecord: Record<defines.entity_status, EntityStatus> = {
+  export function collisionBox(collision_box: BoundingBoxRead): MapBoundingBox {
+    return {
+      leftTop: Converters.entityPosition(collision_box.left_top),
+      rightBottom: Converters.entityPosition(collision_box.right_bottom),
+    }
+  }
+
+
+  const EntityStatusRecord: Record<defines.entity_status, FactorioEntityStatus> = {
     //@formatter:off
-    [ defines.entity_status.cant_divide_segments             ]: EntityStatus.CANT_DIVIDE_SEGMENTS,
-    [ defines.entity_status.charging                         ]: EntityStatus.CHARGING,
-    [ defines.entity_status.closed_by_circuit_network        ]: EntityStatus.CLOSED_BY_CIRCUIT_NETWORK,
-    [ defines.entity_status.disabled                         ]: EntityStatus.DISABLED,
-    [ defines.entity_status.disabled_by_control_behavior     ]: EntityStatus.DISABLED_BY_CONTROL_BEHAVIOR,
-    [ defines.entity_status.disabled_by_script               ]: EntityStatus.DISABLED_BY_SCRIPT,
-    [ defines.entity_status.discharging                      ]: EntityStatus.DISCHARGING,
-    [ defines.entity_status.fluid_ingredient_shortage        ]: EntityStatus.FLUID_INGREDIENT_SHORTAGE,
-    [ defines.entity_status.fully_charged                    ]: EntityStatus.FULLY_CHARGED,
-    [ defines.entity_status.full_output                      ]: EntityStatus.FULL_OUTPUT,
-    [ defines.entity_status.item_ingredient_shortage         ]: EntityStatus.ITEM_INGREDIENT_SHORTAGE,
-    [ defines.entity_status.launching_rocket                 ]: EntityStatus.LAUNCHING_ROCKET,
-    [ defines.entity_status.low_input_fluid                  ]: EntityStatus.LOW_INPUT_FLUID,
-    [ defines.entity_status.low_power                        ]: EntityStatus.LOW_POWER,
-    [ defines.entity_status.low_temperature                  ]: EntityStatus.LOW_TEMPERATURE,
-    [ defines.entity_status.marked_for_deconstruction        ]: EntityStatus.MARKED_FOR_DECONSTRUCTION,
-    [ defines.entity_status.missing_required_fluid           ]: EntityStatus.MISSING_REQUIRED_FLUID,
-    [ defines.entity_status.missing_science_packs            ]: EntityStatus.MISSING_SCIENCE_PACKS,
-    [ defines.entity_status.networks_connected               ]: EntityStatus.NETWORKS_CONNECTED,
-    [ defines.entity_status.networks_disconnected            ]: EntityStatus.NETWORKS_DISCONNECTED,
-    [ defines.entity_status.normal                           ]: EntityStatus.NORMAL,
-    [ defines.entity_status.not_connected_to_rail            ]: EntityStatus.NOT_CONNECTED_TO_RAIL,
-    [ defines.entity_status.not_plugged_in_electric_network  ]: EntityStatus.NOT_PLUGGED_IN_ELECTRIC_NETWORK,
-    [ defines.entity_status.no_ammo                          ]: EntityStatus.NO_AMMO,
-    [ defines.entity_status.no_fuel                          ]: EntityStatus.NO_FUEL,
-    [ defines.entity_status.no_ingredients                   ]: EntityStatus.NO_INGREDIENTS,
-    [ defines.entity_status.no_input_fluid                   ]: EntityStatus.NO_INPUT_FLUID,
-    [ defines.entity_status.no_minable_resources             ]: EntityStatus.NO_MINABLE_RESOURCES,
-    [ defines.entity_status.no_modules_to_transmit           ]: EntityStatus.NO_MODULES_TO_TRANSMIT,
-    [ defines.entity_status.no_power                         ]: EntityStatus.NO_POWER,
-    [ defines.entity_status.no_recipe                        ]: EntityStatus.NO_RECIPE,
-    [ defines.entity_status.no_research_in_progress          ]: EntityStatus.NO_RESEARCH_IN_PROGRESS,
-    [ defines.entity_status.opened_by_circuit_network        ]: EntityStatus.OPENED_BY_CIRCUIT_NETWORK,
-    [ defines.entity_status.out_of_logistic_network          ]: EntityStatus.OUT_OF_LOGISTIC_NETWORK,
-    [ defines.entity_status.preparing_rocket_for_launch      ]: EntityStatus.PREPARING_ROCKET_FOR_LAUNCH,
-    [ defines.entity_status.recharging_after_power_outage    ]: EntityStatus.RECHARGING_AFTER_POWER_OUTAGE,
-    [ defines.entity_status.turned_off_during_daytime        ]: EntityStatus.TURNED_OFF_DURING_DAYTIME,
-    [ defines.entity_status.waiting_for_source_items         ]: EntityStatus.WAITING_FOR_SOURCE_ITEMS,
-    [ defines.entity_status.waiting_for_space_in_destination ]: EntityStatus.WAITING_FOR_SPACE_IN_DESTINATION,
-    [ defines.entity_status.waiting_for_target_to_be_built   ]: EntityStatus.WAITING_FOR_TARGET_TO_BE_BUILT,
-    [ defines.entity_status.waiting_for_train                ]: EntityStatus.WAITING_FOR_TRAIN,
-    [ defines.entity_status.waiting_to_launch_rocket         ]: EntityStatus.WAITING_TO_LAUNCH_ROCKET,
-    [ defines.entity_status.working                          ]: EntityStatus.WORKING,
+    [ defines.entity_status.cant_divide_segments             ]: FactorioEntityStatus.CANT_DIVIDE_SEGMENTS,
+    [ defines.entity_status.charging                         ]: FactorioEntityStatus.CHARGING,
+    [ defines.entity_status.closed_by_circuit_network        ]: FactorioEntityStatus.CLOSED_BY_CIRCUIT_NETWORK,
+    [ defines.entity_status.disabled                         ]: FactorioEntityStatus.DISABLED,
+    [ defines.entity_status.disabled_by_control_behavior     ]: FactorioEntityStatus.DISABLED_BY_CONTROL_BEHAVIOR,
+    [ defines.entity_status.disabled_by_script               ]: FactorioEntityStatus.DISABLED_BY_SCRIPT,
+    [ defines.entity_status.discharging                      ]: FactorioEntityStatus.DISCHARGING,
+    [ defines.entity_status.fluid_ingredient_shortage        ]: FactorioEntityStatus.FLUID_INGREDIENT_SHORTAGE,
+    [ defines.entity_status.fully_charged                    ]: FactorioEntityStatus.FULLY_CHARGED,
+    [ defines.entity_status.full_output                      ]: FactorioEntityStatus.FULL_OUTPUT,
+    [ defines.entity_status.item_ingredient_shortage         ]: FactorioEntityStatus.ITEM_INGREDIENT_SHORTAGE,
+    [ defines.entity_status.launching_rocket                 ]: FactorioEntityStatus.LAUNCHING_ROCKET,
+    [ defines.entity_status.low_input_fluid                  ]: FactorioEntityStatus.LOW_INPUT_FLUID,
+    [ defines.entity_status.low_power                        ]: FactorioEntityStatus.LOW_POWER,
+    [ defines.entity_status.low_temperature                  ]: FactorioEntityStatus.LOW_TEMPERATURE,
+    [ defines.entity_status.marked_for_deconstruction        ]: FactorioEntityStatus.MARKED_FOR_DECONSTRUCTION,
+    [ defines.entity_status.missing_required_fluid           ]: FactorioEntityStatus.MISSING_REQUIRED_FLUID,
+    [ defines.entity_status.missing_science_packs            ]: FactorioEntityStatus.MISSING_SCIENCE_PACKS,
+    [ defines.entity_status.networks_connected               ]: FactorioEntityStatus.NETWORKS_CONNECTED,
+    [ defines.entity_status.networks_disconnected            ]: FactorioEntityStatus.NETWORKS_DISCONNECTED,
+    [ defines.entity_status.normal                           ]: FactorioEntityStatus.NORMAL,
+    [ defines.entity_status.not_connected_to_rail            ]: FactorioEntityStatus.NOT_CONNECTED_TO_RAIL,
+    [ defines.entity_status.not_plugged_in_electric_network  ]: FactorioEntityStatus.NOT_PLUGGED_IN_ELECTRIC_NETWORK,
+    [ defines.entity_status.no_ammo                          ]: FactorioEntityStatus.NO_AMMO,
+    [ defines.entity_status.no_fuel                          ]: FactorioEntityStatus.NO_FUEL,
+    [ defines.entity_status.no_ingredients                   ]: FactorioEntityStatus.NO_INGREDIENTS,
+    [ defines.entity_status.no_input_fluid                   ]: FactorioEntityStatus.NO_INPUT_FLUID,
+    [ defines.entity_status.no_minable_resources             ]: FactorioEntityStatus.NO_MINABLE_RESOURCES,
+    [ defines.entity_status.no_modules_to_transmit           ]: FactorioEntityStatus.NO_MODULES_TO_TRANSMIT,
+    [ defines.entity_status.no_power                         ]: FactorioEntityStatus.NO_POWER,
+    [ defines.entity_status.no_recipe                        ]: FactorioEntityStatus.NO_RECIPE,
+    [ defines.entity_status.no_research_in_progress          ]: FactorioEntityStatus.NO_RESEARCH_IN_PROGRESS,
+    [ defines.entity_status.opened_by_circuit_network        ]: FactorioEntityStatus.OPENED_BY_CIRCUIT_NETWORK,
+    [ defines.entity_status.out_of_logistic_network          ]: FactorioEntityStatus.OUT_OF_LOGISTIC_NETWORK,
+    [ defines.entity_status.preparing_rocket_for_launch      ]: FactorioEntityStatus.PREPARING_ROCKET_FOR_LAUNCH,
+    [ defines.entity_status.recharging_after_power_outage    ]: FactorioEntityStatus.RECHARGING_AFTER_POWER_OUTAGE,
+    [ defines.entity_status.turned_off_during_daytime        ]: FactorioEntityStatus.TURNED_OFF_DURING_DAYTIME,
+    [ defines.entity_status.waiting_for_source_items         ]: FactorioEntityStatus.WAITING_FOR_SOURCE_ITEMS,
+    [ defines.entity_status.waiting_for_space_in_destination ]: FactorioEntityStatus.WAITING_FOR_SPACE_IN_DESTINATION,
+    [ defines.entity_status.waiting_for_target_to_be_built   ]: FactorioEntityStatus.WAITING_FOR_TARGET_TO_BE_BUILT,
+    [ defines.entity_status.waiting_for_train                ]: FactorioEntityStatus.WAITING_FOR_TRAIN,
+    [ defines.entity_status.waiting_to_launch_rocket         ]: FactorioEntityStatus.WAITING_TO_LAUNCH_ROCKET,
+    [ defines.entity_status.working                          ]: FactorioEntityStatus.WORKING,
     //@formatter:on
   }
 }

modules/events-mod/src/main/typescript/main/events/handlers/EntityUpdateHandler.ts

@@ -1,8 +1,9 @@
 import {
   EventName,
-  FactorioEntityData,
+  FactorioEntityUpdateEntity,
   KafkatorioPacketData,
-  MapChunkPosition, PrototypeId,
+  MapChunkPosition,
+  PrototypeId,
   Tick
 } from "../../../generated/kafkatorio-schema";
 import {Converters} from "../converters";
@@ -19,29 +20,34 @@ export class EntityUpdatesHandler {
       return
     }
 
-    const tilesByChunk: LuaTable<MapChunkPosition, OldTileAndPosition> = new LuaTable<MapChunkPosition, OldTileAndPosition>()
+    const tilesByChunk: LuaTable<MapChunkPosition, OldTileAndPosition[]> = new LuaTable<MapChunkPosition, OldTileAndPosition[]>()
 
     for (const tile of event.tiles) {
       const chunkPosition = Converters.tilePositionToChunkPosition(tile.position)
-      tilesByChunk.set(chunkPosition, tile)
+      if (!tilesByChunk.has(chunkPosition)) {
+        tilesByChunk.set(chunkPosition, [])
+      }
+      const tiles = tilesByChunk.get(chunkPosition)
+      tiles[tiles.length] = tile
+      tilesByChunk.set(chunkPosition, tiles)
     }
 
     for (const [chunkPos, tiles] of pairs(tilesByChunk)) {
 
-      const entities: FactorioEntityData.Resource[] = []
+      const entities: LuaEntity[] = []
 
-      for (const tile in tiles) {
-        const resources: FactorioEntityData.Resource[] = EntityUpdatesHandler.getTileResourceEntities(surface, chunkPos)
-        for (const resource of resources) {
+      for (const tile of tiles) {
+        const tileEntities = EntityUpdatesHandler.getTileResourceEntities(surface, tile.position)
+        for (const resource of tileEntities) {
           entities[entities.length] = resource
         }
       }
 
-      EntityUpdatesHandler.throttleResourceEntitiesUpdate(
+      EntityUpdatesHandler.throttleResourcesUpdate(
           Converters.eventNameString(event.name),
           event.tick as Tick,
           event.surface_index,
-          [chunkPos[0], chunkPos[1]],
+          chunkPos,
           entities,
       )
     }
@@ -64,11 +70,11 @@ export class EntityUpdatesHandler {
 
   handleChunkGeneratedEvent(event: OnChunkGeneratedEvent) {
     const entities = EntityUpdatesHandler.getAreaResourceEntities(event.surface, event.area)
-    EntityUpdatesHandler.throttleResourceEntitiesUpdate(
+    EntityUpdatesHandler.throttleResourcesUpdate(
         Converters.eventNameString(event.name),
         event.tick as Tick,
         event.surface.index,
-        [event.position.x, event.position.y],
+        Converters.chunkPosition(event.position),
         entities,
     )
   }
@@ -88,22 +94,16 @@ export class EntityUpdatesHandler {
       chunkPosition: Converters.mapPositionToChunkPosition(entity.position)
     }
 
-    const entityUpdate: FactorioEntityData.Standard = {
-      type: FactorioEntityData.Type.Standard,
-      protoId: Converters.prototypeIdEntity(entity),
+    const entityUpdate: FactorioEntityUpdateEntity = {
+      unitNumber: entity.unit_number,
       graphicsVariation: entity.graphics_variation,
       health: entity.health,
       isActive: entity.active,
       isRotatable: entity.rotatable,
       lastUser: entity.last_user?.index,
-      localisedDescription: null,
-      localisedName: null,
-      position: [entity.position.x, entity.position.y],
       status: Converters.entityStatus(entity.status),
     }
 
-    const entityKey: string = `${entity.unit_number}`
-
     EventDataCache.throttle<KafkatorioPacketData.MapChunkEntityUpdate>(
         entityUpdateKey,
         KafkatorioPacketData.Type.MapChunkEntityUpdate,
@@ -113,42 +113,44 @@ export class EntityUpdatesHandler {
           data.events[eventName] ??= []
           data.events[eventName].push(event.tick)
 
-          data.distinctEntities ??= {}
-          data.distinctEntities[entityKey] = entityUpdate
+          data.entitiesXY ??= {}
+          data.entitiesXY[`${entity.position.x}`] ??= {}
+          data.entitiesXY[`${entity.position.x}`][`${entity.position.y}`] ??= entityUpdate
         }
     )
   }
 
 
-  private static throttleResourceEntitiesUpdate(
+  private static throttleResourcesUpdate(
       eventName: EventName,
       eventTick: Tick,
       surfaceIndex: SurfaceIndex,
       chunkPosition: MapChunkPosition,
-      entities: FactorioEntityData.Resource[],
+      entities: LuaEntity[],
   ) {
 
     // first group by prototype ID
-    const entitiesByProtoId: { [protoId: PrototypeId]: FactorioEntityData.Resource[] } = {}
+    const entitiesByProtoId: { [protoId: PrototypeId]: LuaEntity[] } = {}
 
     for (const entity of entities) {
-      entitiesByProtoId[entity.protoId] ??= []
-      entitiesByProtoId[entity.protoId].push(entity)
+      const protoId = Converters.prototypeIdEntity(entity)
+      entitiesByProtoId[protoId] ??= []
+      entitiesByProtoId[protoId].push(entity)
     }
 
     for (const [protoId, resourceEntities] of pairs(entitiesByProtoId)) {
 
-      // create an update for each prototype
+      // create an update for each prototype ID
 
-      const entityUpdateKey: KafkatorioPacketData.MapChunkEntityUpdate["key"] = {
+      const entityUpdateKey: KafkatorioPacketData.MapChunkResourceUpdate["key"] = {
         protoId: protoId,
         surfaceIndex: surfaceIndex,
-        chunkPosition: [chunkPosition[0], chunkPosition[1]],
+        chunkPosition: chunkPosition,
       }
 
-      EventDataCache.throttle<KafkatorioPacketData.MapChunkEntityUpdate>(
+      EventDataCache.throttle<KafkatorioPacketData.MapChunkResourceUpdate>(
           entityUpdateKey,
-          KafkatorioPacketData.Type.MapChunkEntityUpdate,
+          KafkatorioPacketData.Type.MapChunkResourceUpdate,
           data => {
 
             data.events ??= {}
@@ -156,11 +158,13 @@ export class EntityUpdatesHandler {
             data.events[eventName].push(eventTick)
 
             for (const entity of resourceEntities) {
-              const [x, y] = entity.position
-              const entityKey: string = `${x},${y}`
+              const resourceUpdate = Converters.convertResourceEntity(entity)
 
-              data.distinctEntities ??= {}
-              data.distinctEntities[entityKey] = entity
+              if (resourceUpdate != null) {
+                data.resourcesXY ??= {}
+                data.resourcesXY[`${entity.position.x}`] ??= {}
+                data.resourcesXY[`${entity.position.x}`][`${entity.position.y}`] = resourceUpdate
+              }
             }
           }
       )
@@ -173,24 +177,12 @@ export class EntityUpdatesHandler {
   private static getTileResourceEntities(
       surface: LuaSurface,
       mapPosition: MapPosition,
-  ): FactorioEntityData.Resource[] {
-
-    const rawEntities = surface.find_entities_filtered({
+  ): LuaEntity[] {
+    return surface.find_entities_filtered({
           position: mapPosition,
           collision_mask: "resource-layer",
         }
     )
-
-    const resources: FactorioEntityData.Resource[] = []
-
-    for (const rawEntity of rawEntities) {
-      const entity = Converters.convertResourceEntity(rawEntity)
-      if (entity != null) {
-        resources[resources.length] = entity
-      }
-    }
-
-    return resources
   }
 
 
@@ -198,24 +190,12 @@ export class EntityUpdatesHandler {
   private static getAreaResourceEntities(
       surface: LuaSurface,
       area: BoundingBoxTable,
-  ): FactorioEntityData.Resource[] {
-
-    const rawEntities = surface.find_entities_filtered({
+  ): LuaEntity[] {
+    return surface.find_entities_filtered({
           area: area,
           collision_mask: "resource-layer",
         }
     )
-
-    const resources: FactorioEntityData.Resource[] = []
-
-    for (const rawEntity of rawEntities) {
-      const entity = Converters.convertResourceEntity(rawEntity)
-      if (entity != null) {
-        resources[resources.length] = entity
-      }
-    }
-
-    return resources
   }
 }
 

modules/events-mod/src/main/typescript/main/events/handlers/MapChunkUpdateHandler.ts

@@ -22,7 +22,7 @@ class MapChunkUpdateHandler {
 
     MapChunkUpdateHandler.mapTilesUpdateDebounce(
         event.surface,
-        Converters.convertMapTablePosition(event.position),
+        Converters.chunkPosition(event.position),
         tiles,
         event,
         undefined,
@@ -136,7 +136,7 @@ class MapChunkUpdateHandler {
     for (const position of event.positions) {
       const key: PacketKey<KafkatorioPacketData.MapChunkTileUpdate> = {
         surfaceIndex: surface.index,
-        chunkPosition: [position.x, position.y],
+        chunkPosition: Converters.chunkPosition(position),
       }
 
       EventUpdates.debounce<KafkatorioPacketData.MapChunkTileUpdate>(
@@ -219,10 +219,7 @@ class MapChunkUpdateHandler {
 
     const mapChunkPositionToTiles = new Map<MapChunkPosition, TileRead[]>()
     for (const tile of tiles) {
-      const chunkPosition: MapChunkPosition = [
-        math.floor((tile.position.x / 32)),
-        math.floor((tile.position.y / 32)),
-      ]
+      const chunkPosition = Converters.tilePositionToChunkPosition(tile.position)
       if (!mapChunkPositionToTiles.has(chunkPosition)) {
         mapChunkPositionToTiles.set(chunkPosition, [])
       }

modules/events-mod/src/main/typescript/main/events/prototypeUpdates.ts

@@ -1,4 +1,3 @@
-import PacketEmitter from "../emitting/PacketEmitter";
 import {
   EntityItemGroup,
   FactorioPrototype,
@@ -16,7 +15,6 @@ export function emitPrototypes() {
   const mapTileProtos: PrototypesByType<FactorioPrototype.MapTile> = getMapTilePrototypes()
   emit(mapTileProtos)
 
-
   const entityProtos: PrototypesByType<FactorioPrototype.Entity> = getEntityPrototypes()
   emit(entityProtos)
 }
@@ -58,7 +56,7 @@ function getEntityPrototypes(): PrototypesByType<FactorioPrototype.Entity> {
   const protos: PrototypesByType<FactorioPrototype.Entity> = {}
 
   for (const [, entity] of game.entity_prototypes) {
-    const key: string = `${entity.group.name}/${entity.subgroup.name}`
+    const key: string = `${entity.group.name}/${entity.subgroup.name}/${entity.type}`
 
     protos[key] ??= []
 
@@ -73,7 +71,9 @@ function getEntityPrototypes(): PrototypesByType<FactorioPrototype.Entity> {
       isEntityWithOwner: entity.is_entity_with_owner,
       isMilitaryTarget: entity.is_military_target,
       maxHealth: entity.max_health,
+      collisionBox: Converters.collisionBox(entity.collision_box),
     }
+    entity.infinite_resource
 
     if (entity.map_color != undefined) {
       entityProto.mapColour = Converters.mapColour(entity.map_color)

modules/events-mod/src/main/typescript/main/events/registerEventListeners.ts

@@ -3,10 +3,10 @@ import {Converters} from "./converters";
 import {handleSurfaceUpdate} from "./surfaceUpdates";
 import EventCacheService from "../emitting/EventDataCache";
 import MapChunkUpdate from "./handlers/MapChunkUpdateHandler";
-import EventDataQueue from "../queue/queue";
 import KafkatorioPacketQueue from "../emitting/KafkatorioPacketQueue";
 import PacketEmitter from "../emitting/PacketEmitter";
 import {KafkatorioPacketData} from "../../generated/kafkatorio-schema";
+import EventDataQueue from "../emitting/EventDataQueue";
 
 
 script.on_event(defines.events.on_tick, (event: OnTickEvent) => {

modules/events-mod/src/main/typescript/main/global-init.ts

@@ -1,5 +1,5 @@
 import EventUpdates from "./emitting/EventDataCache";
-import {EventDataQueueManager} from "./queue/queue";
+import {EventDataQueueManager} from "./emitting/EventDataQueue";
 
 
 export function initGlobal(force: boolean = false) {

modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts

@@ -8,6 +8,7 @@ export class KafkatorioSettingsConfig {
     "kafkatorio.packet.keyed.PlayerUpdate": 0.5,
     "kafkatorio.packet.keyed.MapChunkTileUpdate": 30,
     "kafkatorio.packet.keyed.MapChunkEntityUpdate": 30,
+    "kafkatorio.packet.keyed.MapChunkResourceUpdate": 30,
     "kafkatorio.packet.keyed.EntityUpdate": 5,
   }
 

modules/events-mod/tsconfig.json

@@ -24,20 +24,20 @@
     "forceConsistentCasingInFileNames": true,
     "listEmittedFiles": true,
     "listFiles": true,
+    "noFallthroughCasesInSwitch": true,
     "noImplicitAny": true,
+    "noImplicitOverride": true,
+    "noImplicitReturns": true,
     "noImplicitThis": true,
     "sourceMap": false,
     "strict": true,
     "strictFunctionTypes": true,
     "strictNullChecks": true,
-    "traceResolution": true,
-    "noImplicitReturns": true,
-    "noImplicitOverride": true,
-    "noFallthroughCasesInSwitch": true
+    "traceResolution": false
   },
   "tstl": {
     "luaTarget": "5.2",
-    "tstlVerbose": true,
+    "tstlVerbose": false,
     "noHeader": true,
     "noImplicitSelf": true,
     "sourceMapTraceback": false,

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/eventsServer.kt

@@ -20,12 +20,10 @@ suspend fun main(): Unit = coroutineScope {
     startWebServer(appProps, websocketServer)
   }
 
-
   launch {
     syslogSocketServer.start()
   }
 
-
   launch {
     eventsProcessor(appProps, syslogSocketServer, websocketServer)
   }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/DebounceProcessor.kt

@@ -20,19 +20,15 @@ import org.apache.kafka.streams.state.StoreBuilder
 import org.apache.kafka.streams.state.Stores
 import org.apache.kafka.streams.state.TimestampedKeyValueStore
 import org.apache.kafka.streams.state.ValueAndTimestamp
-import java.time.Duration as DurationJvm
-import kotlin.time.Duration as DurationKt
+import kotlin.time.Duration
 
 
 class DebounceProcessor<K, V>(
-  private val inactivityDuration: DurationKt,
-  checkInterval: DurationKt,
+  private val inactivityDuration: Duration,
+  private val checkInterval: Duration,
   private val storeName: String,
 ) : Processor<K, V, K, V> {
 
-
-  private val checkInterval: DurationJvm = checkInterval.toJavaDuration()
-
   private lateinit var state: State<K, V>
 
 
@@ -70,7 +66,7 @@ class DebounceProcessor<K, V>(
   ) : State<K, V> {
     val store: TimestampedKeyValueStore<K, V> = context.getStateStore(storeName)
     val schedule: Cancellable = context.schedule(
-      checkInterval,
+      checkInterval.toJavaDuration(),
       PunctuationType.WALL_CLOCK_TIME,
       ::checkRecords
     )
@@ -95,8 +91,8 @@ class DebounceProcessor<K, V>(
       sourceTopic: String,
       sinkTopic: String,
 
-      inactivityDuration: DurationKt,
-      checkInterval: DurationKt = inactivityDuration / 10,
+      inactivityDuration: Duration,
+      checkInterval: Duration = inactivityDuration / 10,
       keySerde: Serde<K>,
       valueSerde: Serde<V>,
       storeName: String = "$namePrefix.debounce-store",
@@ -131,8 +127,8 @@ class DebounceProcessor<K, V>(
     }
 
     class Supplier<K, V>(
-      private val inactivityDuration: DurationKt,
-      private val checkInterval: DurationKt,
+      private val inactivityDuration: Duration,
+      private val checkInterval: Duration,
       private val storeName: String,
       private val keySerde: Serde<K>,
       private val valueSerde: Serde<V>,

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/KafkatorioTopology.kt

@@ -1,12 +1,24 @@
 package dev.adamko.kafkatorio.server.processor
 
 import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
+import dev.adamko.kafkatorio.schema.common.FactorioServerId
+import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
+import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
+import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
+import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
+import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
 import dev.adamko.kafkatorio.server.config.ApplicationProperties
+import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
 import dev.adamko.kafkatorio.server.processor.topology.broadcastToWebsocket
+import dev.adamko.kafkatorio.server.processor.topology.colourEntityChunks
 import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
+import dev.adamko.kafkatorio.server.processor.topology.convertEntityUpdateToServerMapChunkTiles
+import dev.adamko.kafkatorio.server.processor.topology.convertResourceUpdateToServerMapChunkTiles
 import dev.adamko.kafkatorio.server.processor.topology.factorioServerPacketStream
 import dev.adamko.kafkatorio.server.processor.topology.saveMapTiles
 import dev.adamko.kafkatorio.server.processor.topology.splitFactorioServerPacketStream
+import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
 import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
 import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
 import java.util.Properties
@@ -35,6 +47,7 @@ import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.StreamsConfig
 import org.apache.kafka.streams.Topology
 import org.apache.kafka.streams.TopologyDescription
+import org.apache.kafka.streams.kstream.KStream
 
 
 internal class KafkatorioTopology(
@@ -50,6 +63,19 @@ internal class KafkatorioTopology(
     splitPackets()
 
     groupTilesMapChunks()
+
+    groupEntitiesMapChunks(
+      layer = ServerMapTileLayer.BUILDING,
+      colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES,
+      colouredChunkStateTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
+    )
+
+    groupEntitiesMapChunks(
+      layer = ServerMapTileLayer.RESOURCE,
+      colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES,
+      colouredChunkStateTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
+    )
+
     saveTiles()
 
     websocketBroadcaster()
@@ -69,10 +95,53 @@ internal class KafkatorioTopology(
   }
 
 
+  /** Terrain tiles */
   private suspend fun groupTilesMapChunks() {
     val builder = StreamsBuilder()
-    val topology = colourMapChunks(builder)
-    launchTopology("groupTilesMapChunks", topology)
+
+    val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
+      builder.streamPacketData()
+
+    val topology = colourMapChunks(builder, protosStream)
+
+    launchTopology("groupTilesMapChunks.${ServerMapTileLayer.TERRAIN.dir}", topology)
+  }
+
+
+  /** Building and resource tiles */
+  private suspend fun groupEntitiesMapChunks(
+    layer: ServerMapTileLayer,
+    colouredUpdatesStreamTopic: String,
+    colouredChunkStateTopic: String,
+  ) {
+    val builder = StreamsBuilder()
+
+    val entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> =
+      when (layer) {
+        ServerMapTileLayer.TERRAIN  -> return
+
+        ServerMapTileLayer.RESOURCE ->
+          builder.streamPacketData<MapChunkResourceUpdate>()
+            .convertResourceUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
+
+        ServerMapTileLayer.BUILDING ->
+          builder.streamPacketData<MapChunkEntityUpdate>()
+            .convertEntityUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
+      }
+
+    val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
+      builder.streamPacketData()
+
+    val topology = colourEntityChunks(
+      builder,
+      entityChunksStream,
+      protosStream,
+      layer,
+      colouredUpdatesStreamTopic,
+      colouredChunkStateTopic,
+    )
+
+    launchTopology("groupTilesMapChunks.${layer.dir}", topology)
   }
 
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/kafkatorioTopics.kt

@@ -5,7 +5,9 @@ import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate
 import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate
 import dev.adamko.kafkatorio.schema.packets.EntityUpdate
 import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
+import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
 import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
+import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
 import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
 import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
 import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
@@ -17,14 +19,21 @@ private const val DOMAIN = "kafkatorio"
 
 const val TOPIC_SRC_SERVER_LOG = "$DOMAIN.src.server-log"
 
-const val TOPIC_GROUPED_MAP_CHUNKS_STATE = "$DOMAIN.state.map-chunks.grouped"
+//const val TOPIC_GROUPED_MAP_CHUNKS_STATE = "$DOMAIN.state.map-chunks.grouped"
 
-const val TOPIC_MAP_CHUNK_COLOURED_032_STATE = "$DOMAIN.state.map-chunk-032.colour"
+const val TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES =
+  "$DOMAIN.state-updates.map-chunk-032.terrain.colour"
+const val TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES =
+  "$DOMAIN.state-updates.map-chunk-032.resource.colour"
+const val TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES =
+  "$DOMAIN.state-updates.map-chunk-032.building.colour"
 
-const val TOPIC_MAP_CHUNK_COLOURED_STATE = "$DOMAIN.state.map-chunk.colour"
+const val TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE = "$DOMAIN.state.map-chunk.terrain.colour"
+const val TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE = "$DOMAIN.state.map-chunk.resource.colour"
+const val TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE = "$DOMAIN.state.map-chunk.building.colour"
 
-const val TOPIC_SUBDIVIDED_MAP_TILES = "$DOMAIN.map-tiles.subdivided"
-const val TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED = "$TOPIC_SUBDIVIDED_MAP_TILES.debounced"
+//const val TOPIC_SUBDIVIDED_MAP_TILES = "$DOMAIN.map-tiles.subdivided"
+//const val TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED = "$TOPIC_SUBDIVIDED_MAP_TILES.debounced"
 
 
 /** Broadcast these events out, via websocket */
@@ -33,14 +42,18 @@ const val TOPIC_BROADCAST_TO_WEBSOCKET = "$DOMAIN.broadcast.websocket"
 fun allTopics(): Set<String> = buildSet {
   add(TOPIC_SRC_SERVER_LOG)
 
-  add(TOPIC_GROUPED_MAP_CHUNKS_STATE)
+//  add(TOPIC_GROUPED_MAP_CHUNKS_STATE)
 
-  add(TOPIC_MAP_CHUNK_COLOURED_032_STATE)
+  add(TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES)
+  add(TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES)
+  add(TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES)
 
-  add(TOPIC_MAP_CHUNK_COLOURED_STATE)
+  add(TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE)
+  add(TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE)
+  add(TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE)
 
-  add(TOPIC_SUBDIVIDED_MAP_TILES)
-  add(TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED)
+//  add(TOPIC_SUBDIVIDED_MAP_TILES)
+//  add(TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED)
 
   add(TOPIC_BROADCAST_TO_WEBSOCKET)
 
@@ -57,8 +70,9 @@ val KafkatorioPacketData.topicName: String
     is EntityUpdate              -> packetTopicNames.getValue(EntityUpdate::class)
     is MapChunkTileUpdate        -> packetTopicNames.getValue(MapChunkTileUpdate::class)
     is PlayerUpdate              -> packetTopicNames.getValue(PlayerUpdate::class)
-    is KafkatorioPacketData.Error -> packetTopicNames.getValue(KafkatorioPacketData.Error::class)
+    is KafkatorioPacketDataError -> packetTopicNames.getValue(KafkatorioPacketDataError::class)
     is MapChunkEntityUpdate      -> packetTopicNames.getValue(MapChunkEntityUpdate::class)
+    is MapChunkResourceUpdate    -> packetTopicNames.getValue(MapChunkResourceUpdate::class)
   }
 
 
@@ -72,7 +86,8 @@ private val packetTopicNames: Map<KClass<out KafkatorioPacketData>, String> = ma
   MapChunkTileUpdate::class to "map-chunk-tile",
   PlayerUpdate::class to "player",
   MapChunkEntityUpdate::class to "map-chunk-entity",
-  KafkatorioPacketData.Error::class to "error",
+  MapChunkResourceUpdate::class to "map-chunk-resource",
+  KafkatorioPacketDataError::class to "error",
 ).mapValues { (_, v) ->
   "$DOMAIN.packet.$v"
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/colourMapChunks.kt

@@ -1,17 +1,20 @@
 package dev.adamko.kafkatorio.server.processor.topology
 
 import dev.adamko.kafkatorio.library.kxsBinary
+import dev.adamko.kafkatorio.library.resize
+import dev.adamko.kafkatorio.library.toMapChunkPosition
 import dev.adamko.kafkatorio.schema.common.ChunkSize
 import dev.adamko.kafkatorio.schema.common.ColourHex
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
+import dev.adamko.kafkatorio.schema.common.MapTilePosition
+import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
 import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
-import dev.adamko.kafkatorio.schema.common.leftTopTile
-import dev.adamko.kafkatorio.schema.common.toMapChunkPosition
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
 import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
 import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
 import dev.adamko.kafkatorio.server.processor.DebounceProcessor.Companion.addDebounceProcessor
-import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_COLOURED_032_STATE
-import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_COLOURED_STATE
+import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES
+import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
 import dev.adamko.kotka.extensions.consumedAs
 import dev.adamko.kotka.extensions.groupedAs
 import dev.adamko.kotka.extensions.materializedAs
@@ -19,8 +22,8 @@ import dev.adamko.kotka.extensions.producedAs
 import dev.adamko.kotka.extensions.repartitionedAs
 import dev.adamko.kotka.extensions.stream
 import dev.adamko.kotka.extensions.streams.filter
+import dev.adamko.kotka.extensions.streams.flatMap
 import dev.adamko.kotka.extensions.streams.map
-import dev.adamko.kotka.extensions.streams.mapValues
 import dev.adamko.kotka.extensions.streams.peek
 import dev.adamko.kotka.extensions.streams.reduce
 import dev.adamko.kotka.extensions.tableJoined
@@ -36,27 +39,27 @@ import org.apache.kafka.streams.kstream.KStream
 import org.apache.kafka.streams.kstream.KTable
 
 
-private const val pid: String = "colourMapChunks"
-
-
 /** Add colour to each [MapChunkTileUpdate] */
-fun colourMapChunks(builder: StreamsBuilder): Topology {
+fun colourMapChunks(
+  builder: StreamsBuilder,
+  protosStream: KStream<FactorioServerId, PrototypesUpdate>
+): Topology {
+  val pid = "colourMapChunks.terrain"
 
   val mapChunksStream: KStream<FactorioServerId, MapChunkTileUpdate> = builder.streamPacketData()
-  val protosStream: KStream<FactorioServerId, PrototypesUpdate> = builder.streamPacketData()
 
   val protosTable: KTable<FactorioServerId, TileColourDict> =
     protosStream.createTilePrototypeTable()
 
   val chunkTilesColouredTable032: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
     mapChunksStream
-      .convertToServerMapChunkTiles()
-      .reduceServerMapChunkTilesToTable()
-      .enrichWithColourData(protosTable)
+      .convertToServerMapChunkTiles(pid)
+      .reduceServerMapChunkTilesToTable(pid)
+      .enrichWithColourData(pid, protosTable)
 
   chunkTilesColouredTable032.streamMapChunkColouredTo(
-    ChunkSize.CHUNK_032,
-    TOPIC_MAP_CHUNK_COLOURED_032_STATE,
+    "$pid.${ChunkSize.CHUNK_032}",
+    TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES,
   )
 
   val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
@@ -66,7 +69,7 @@ fun colourMapChunks(builder: StreamsBuilder): Topology {
         kxsBinary.serde(ServerMapChunkId.serializer()),
         kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
       ),
-      TOPIC_MAP_CHUNK_COLOURED_STATE,
+      TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
     )
 
   mapOf(
@@ -77,71 +80,78 @@ fun colourMapChunks(builder: StreamsBuilder): Topology {
 
     ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
     ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
-
   ).forEach { (from, to) ->
     chunkTilesColoured
-      .filter("$pid.combine-chunks.filter-$from-to-$to") { key, _ ->
+      .filter("$pid.combine-chunks.filter-$from-to-$to") { key: ServerMapChunkId, _: ServerMapChunkTiles<ColourHex> ->
         key.chunkSize == from
-      }.groupByChunkPosition(to)
-      .streamMapChunkColouredTo(to, TOPIC_MAP_CHUNK_COLOURED_STATE)
+      }.groupByChunkPosition("$pid.from-$from-to-$to", to)
+      .streamMapChunkColouredTo(
+        "$pid.from-$from-to-$to",
+        TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
+      )
   }
 
   return builder.build()
     .addDebounceProcessor(
-      namePrefix = "groupTilesMapChunks",
-      sourceTopic = TOPIC_MAP_CHUNK_COLOURED_032_STATE,
-      sinkTopic = TOPIC_MAP_CHUNK_COLOURED_STATE,
-      inactivityDuration = 5.seconds,
+      namePrefix = "groupMapChunks.terrain",
+      sourceTopic = TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES,
+      sinkTopic = TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
+      inactivityDuration = 15.seconds,
       keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
       valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
     )
 }
 
 
-private fun KStream<FactorioServerId, MapChunkTileUpdate>.convertToServerMapChunkTiles()
-    : KStream<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>> {
+private fun KStream<FactorioServerId, MapChunkTileUpdate>.convertToServerMapChunkTiles(
+  callerPid: String
+): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
 
-  val pid = "$pid.convertToMapTiles"
+  val pid = "$callerPid.convertToMapTiles"
 
   return map("$pid.map") { serverId: FactorioServerId, update: MapChunkTileUpdate ->
 
     val mapTileList = update.tileDictionary?.toMapTileList() ?: emptyList()
 
-    val (validMapTilesList, invalidMapTilesList) = mapTileList.partition {
+    val (validMapTiles, invalidMapTiles) = mapTileList.partition {
       it.position in update.key.chunkPosition
     }
 
-    if (invalidMapTilesList.isNotEmpty()) {
-      println("WARNING [reduceMapTilesToTable] MapChunkTileUpdate contained out-of-bounds tiles $invalidMapTilesList")
+    if (invalidMapTiles.isNotEmpty()) {
+      println(
+        "WARNING [$pid] MapChunkTileUpdate contained ${invalidMapTiles.size} out-of-bounds tiles, " +
+            "key: ${update.key},\n" +
+            "invalid tiles:" + invalidMapTiles.joinToString(limit = 10)
+      )
     }
 
-    val validMapTilesToProtoHashCode =
-      validMapTilesList.associate { it.position to TileProtoHashCode(it) }
+    val validMapTilesToProtoHashCode = validMapTiles.associate {
+      it.position to PrototypeHashCode(it.protoId)
+    }
 
     val chunkId = ServerMapChunkId(
       serverId = serverId,
+      layer = ServerMapTileLayer.TERRAIN,
       chunkPosition = update.key.chunkPosition,
       surfaceIndex = update.key.surfaceIndex,
-      chunkSize = ChunkSize.CHUNK_032,
     )
 
-    val chunkTiles = ServerMapChunkTiles(chunkId, validMapTilesToProtoHashCode)
-
-    chunkId to chunkTiles
+    chunkId to ServerMapChunkTiles(chunkId, validMapTilesToProtoHashCode)
   }
 }
 
 
-private fun KStream<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.reduceServerMapChunkTilesToTable()
-    : KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>> {
+fun KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.reduceServerMapChunkTilesToTable(
+  callerPid: String,
+): KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
 
-  val pid = "$pid.reduceServerMapChunkTilesToTable"
+  val pid = "$callerPid.reduceServerMapChunkTilesToTable"
 
   return repartition(
     repartitionedAs(
       "$pid.pre-table-repartition",
       kxsBinary.serde<ServerMapChunkId>(),
-      kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
+      kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
       // force, otherwise KTable-KTable FK join doesn't work
       numberOfPartitions = 1,
     )
@@ -149,14 +159,14 @@ private fun KStream<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.re
     groupedAs(
       "$pid.group-by-key",
       kxsBinary.serde<ServerMapChunkId>(),
-      kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
+      kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
     )
   ).reduce(
     "$pid.reduce",
     materializedAs(
       "$pid.reduce.store",
       kxsBinary.serde<ServerMapChunkId>(),
-      kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
+      kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
     )
   ) { chunkTiles, otherChunkTiles ->
     chunkTiles + otherChunkTiles
@@ -164,13 +174,14 @@ private fun KStream<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.re
 }
 
 
-private fun KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.enrichWithColourData(
+private fun KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.enrichWithColourData(
+  callerPid: String,
   protosTable: KTable<FactorioServerId, TileColourDict>,
 ): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
 
-  val pid = "$pid.enrichWithColourData"
+  val pid = "$callerPid.enrichWithColourData"
 
-  return this.join(
+  return join(
     other = protosTable,
     tableJoined = tableJoined("$pid.join-tiles-with-prototypes"),
     materialized = materializedAs(
@@ -178,26 +189,30 @@ private fun KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.enr
       kxsBinary.serde(),
       kxsBinary.serde(),
     ),
-    foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<TileProtoHashCode> ->
+    foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<PrototypeHashCode> ->
       chunkTiles.chunkId.serverId
     }
-  ) { tiles: ServerMapChunkTiles<TileProtoHashCode>, colourDict: TileColourDict ->
-//        println("joining tiles:${tiles.map.size} with colourDict:${colourDict.map.size}")
+  ) { tiles: ServerMapChunkTiles<PrototypeHashCode>, colourDict: TileColourDict ->
+    println("joining tiles:${tiles.map.size} with colourDict:${colourDict.size}")
 
-    val missingProtos = mutableSetOf<TileProtoHashCode>()
+    val missingProtos = mutableSetOf<PrototypeHashCode>()
 
-    val tileColours = tiles.map.mapValues { (_, code) ->
-      colourDict.getOrElse(code) {
+    val tileColours: Map<MapTilePosition, ColourHex> =
+      tiles.map.mapNotNull { (position, code) ->
+        when (val colour = colourDict[code]) {
+          null -> {
             missingProtos.add(code)
-        ColourHex.TRANSPARENT
+            null
           }
+          else -> position to colour
         }
+      }.toMap()
 
     if (missingProtos.isNotEmpty()) {
       println("[$pid] missing ${missingProtos.size} tile prototypes: ${missingProtos.joinToString { "${it.code}" }}")
     }
 
-    println("[$pid] Set tile colours for chunk ${tiles.chunkId}")
+    println("[$pid] Set ${tileColours.size} tile colours for chunk ${tiles.chunkId}")
 
     ServerMapChunkTiles(tiles.chunkId, tileColours)
   }
@@ -205,20 +220,16 @@ private fun KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.enr
 
 
 fun KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.streamMapChunkColouredTo(
-  chunkSize: ChunkSize,
+  callerPid: String,
   outputTopic: String,
 ) {
-  val pid = "$pid.output-chunk.${chunkSize.name}"
+  val pid = "$callerPid.output-chunk"
 
   toStream("$pid.stream")
-    .filter("$pid.filter-tiles-not-empty") { _, chunkTiles ->
-      !chunkTiles?.map.isNullOrEmpty()
-    }
-    .mapValues("$pid.map-not-null") { _, chunkTiles ->
-      requireNotNull(chunkTiles)
-    }
-    .peek("$pid.print-group-result") { _, chunkTiles ->
-      println("Grouping map tiles result: ${chunkTiles.chunkId} / size:${chunkTiles.map.size}")
+    .flatMap("$pid.group-by-chunks") { _: ServerMapChunkId, tiles: ServerMapChunkTiles<ColourHex>? ->
+      reChunkTiles(pid, tiles)
+    }.peek("$pid.print-group-result") { _, chunkTiles ->
+      println("[$pid] Grouping map tiles result: ${chunkTiles.chunkId} / size:${chunkTiles.map.size}")
     }.to(
       outputTopic,
       producedAs(
@@ -235,26 +246,22 @@ fun KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.streamMapChunkColou
  *
  * They should already be grouped, but do it again to make sure, and to filter out empty updates.
  */
-private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.groupByChunkPosition(
-  chunkSize: ChunkSize = ChunkSize.CHUNK_032
+fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.groupByChunkPosition(
+  callerPid: String,
+  targetChunkSize: ChunkSize = ChunkSize.CHUNK_032
 ): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
 
-  val pid = "$pid.groupByChunkPosition.${chunkSize.name}"
+  val pid = "$callerPid.groupByChunkPosition.${targetChunkSize.name}"
 
-  return map("$pid.change-chunk-size") { chunkId: ServerMapChunkId, chunkTiles: ServerMapChunkTiles<ColourHex>? ->
+  return flatMap("$pid.change-chunk-size") { _: ServerMapChunkId, chunkTiles: ServerMapChunkTiles<ColourHex>? ->
+//    println("[$pid] changing chunk size for $chunkId")
+    chunkTiles ?: return@flatMap emptyList()
 
-    val newChunkPosition = chunkId.chunkPosition
-      .leftTopTile(chunkId.chunkSize)
-      .toMapChunkPosition(chunkSize)
+    val newChunkPosition = chunkTiles.chunkId.chunkPosition.resize(targetChunkSize)
+    val newId = chunkTiles.chunkId.copy(chunkPosition = newChunkPosition)
+    val newChunkTiles = chunkTiles.copy(chunkId = newId)
 
-    val newId = chunkId.copy(
-      chunkSize = chunkSize,
-      chunkPosition = newChunkPosition,
-    )
-
-    val newChunkTiles = chunkTiles?.map ?: emptyMap()
-
-    newId to ServerMapChunkTiles(chunkId = newId, map = newChunkTiles)
+    reChunkTiles(pid, newChunkTiles)
   }.groupByKey(
     groupedAs(
       "$pid.group-by-key",
@@ -272,3 +279,51 @@ private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.groupByChu
     chunkTiles + otherChunkTiles
   }
 }
+
+
+fun <T : Any> reChunkTiles(
+  pid: String,
+  tiles: ServerMapChunkTiles<T>?,
+): List<Pair<ServerMapChunkId, ServerMapChunkTiles<T>>> {
+
+  return when {
+    tiles == null || tiles.map.isEmpty() -> emptyList()
+
+    tiles.map.keys.all { it in tiles.chunkId.chunkPosition } -> listOf(tiles.chunkId to tiles)
+
+    else -> {
+      val chunkId = tiles.chunkId
+
+      val (boundedTiles, unboundedTiles) = tiles.map.entries.partition { (position, _) ->
+        position in chunkId.chunkPosition
+      }
+
+      if (unboundedTiles.isNotEmpty()) {
+        val unboundedEntities = unboundedTiles.map { (_, entity) -> entity }
+        println(
+          "WARNING [$pid] MapChunkEntityUpdate $chunkId "
+              + "contained ${unboundedEntities.size} out-of-bounds entities "
+              + unboundedEntities.joinToString(limit = 10)
+        )
+      }
+
+      // the existing chunkId and its tiles
+      val boundedTilesChunk =
+        chunkId to ServerMapChunkTiles(chunkId, boundedTiles.associate { it.key to it.value })
+
+      // re-calculate a chunk for the out-of-bounds tiles
+      val chunkedUnboundedTiles = unboundedTiles
+        .map { it.key to it.value }
+        .groupBy { (position, _) ->
+          position.toMapChunkPosition(chunkId.chunkSize)
+        }.map { (newChunkPosition, tiles) ->
+          val newChunkId = chunkId.copy(chunkPosition = newChunkPosition)
+          newChunkId to ServerMapChunkTiles(newChunkId, tiles.toMap())
+        }
+
+      println("[$pid] re-bounded ${unboundedTiles.size} tiles into ${chunkedUnboundedTiles.size} chunks in $chunkId")
+
+      listOf(boundedTilesChunk) + chunkedUnboundedTiles
+    }
+  }
+}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/colourResourceChunks.kt

@@ -0,0 +1,442 @@
+package dev.adamko.kafkatorio.server.processor.topology
+
+import dev.adamko.kafkatorio.library.kxsBinary
+import dev.adamko.kafkatorio.library.toMapTilePosition
+import dev.adamko.kafkatorio.schema.common.ChunkSize
+import dev.adamko.kafkatorio.schema.common.ColourHex
+import dev.adamko.kafkatorio.schema.common.FactorioPrototype
+import dev.adamko.kafkatorio.schema.common.FactorioServerId
+import dev.adamko.kafkatorio.schema.common.MapBoundingBox
+import dev.adamko.kafkatorio.schema.common.MapTilePosition
+import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
+import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
+import dev.adamko.kafkatorio.schema.common.toHex
+import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
+import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
+import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
+import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
+import dev.adamko.kafkatorio.server.processor.DebounceProcessor.Companion.addDebounceProcessor
+import dev.adamko.kotka.extensions.consumedAs
+import dev.adamko.kotka.extensions.materializedAs
+import dev.adamko.kotka.extensions.stream
+import dev.adamko.kotka.extensions.streams.filter
+import dev.adamko.kotka.extensions.streams.flatMap
+import dev.adamko.kotka.extensions.tableJoined
+import dev.adamko.kotka.extensions.tables.join
+import dev.adamko.kotka.kxs.serde
+import kotlin.collections.component1
+import kotlin.collections.component2
+import kotlin.math.ceil
+import kotlin.math.floor
+import kotlin.time.Duration.Companion.seconds
+import org.apache.kafka.streams.StreamsBuilder
+import org.apache.kafka.streams.Topology
+import org.apache.kafka.streams.kstream.KStream
+import org.apache.kafka.streams.kstream.KTable
+
+
+/**
+ * Add colour to each [MapChunkTileUpdate]
+ *
+ * @param[colouredUpdatesStreamTopic] Output topic for the result of converting input updates into 32 sized chunks
+ * @param[colouredChunkStateTopic] Post-debounce state topic, used to recursively combine chunks
+ */
+fun colourEntityChunks(
+  builder: StreamsBuilder,
+  entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>,
+  protosStream: KStream<FactorioServerId, PrototypesUpdate>,
+  layer: ServerMapTileLayer,
+  colouredUpdatesStreamTopic: String,
+  colouredChunkStateTopic: String,
+): Topology {
+  val pid = "colourEntityChunks.${layer.dir}"
+
+  val protosTable: KTable<FactorioServerId, EntityPrototypeMap> =
+    protosStream.createEntityPrototypeMap(pid)
+
+  val chunkEntitiesColouredTable032: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
+    entityChunksStream
+      .reduceServerMapChunkTilesToTable(pid)
+      .enrichWithColourData(pid, protosTable)
+
+  chunkEntitiesColouredTable032.streamMapChunkColouredTo(
+    "$pid.${ChunkSize.CHUNK_032}",
+    colouredUpdatesStreamTopic,
+  )
+
+  val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
+    builder.stream(
+      consumedAs(
+        "$pid.consume.map-chunks-debounced",
+        kxsBinary.serde(ServerMapChunkId.serializer()),
+        kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
+      ),
+      colouredChunkStateTopic,
+    )
+
+  mapOf(
+//    ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
+//    ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
+//    ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
+//    ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
+
+    ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
+    ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
+  ).forEach { (from, to) ->
+    chunkTilesColoured
+      .filter("$pid.combine-chunks.filter-$from-to-$to") { key, _ ->
+        key.chunkSize == from && key.layer == layer
+      }.groupByChunkPosition(pid, to)
+      .streamMapChunkColouredTo(
+        "$pid.from-$from-to-$to",
+        colouredChunkStateTopic,
+      )
+  }
+
+  return builder.build().addDebounceProcessor(
+    namePrefix = "groupMapChunks.${layer.dir}",
+    sourceTopic = colouredUpdatesStreamTopic,
+    sinkTopic = colouredChunkStateTopic,
+    inactivityDuration = 15.seconds,
+    keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
+    valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
+  )
+}
+
+
+fun KStream<FactorioServerId, MapChunkEntityUpdate>.convertEntityUpdateToServerMapChunkTiles(
+  callerPid: String,
+): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
+
+  val pid = "$callerPid.convertToServerMapChunkTiles"
+
+  return flatMap("$pid.map") { serverId: FactorioServerId, update: MapChunkEntityUpdate ->
+
+//    val entities = update.entitiesXY.convert { entity, position ->
+//      FactorioEntityData.Standard(
+//        protoId = update.key.protoId,
+//        position = position,
+//        element = entity,
+//      )
+//    }
+
+//    val (validEntityPositions, unboundedEntityPositions) = entities.map { entity ->
+//      entity.position.toMapTilePosition() to entity
+//    }.partition { (position, _) ->
+//      position in update.key.chunkPosition
+//    }
+//
+//    if (unboundedEntityPositions.isNotEmpty()) {
+//      val unboundedEntities = unboundedEntityPositions.map { (_, entity) -> entity }
+//      println(
+//        "WARNING [convertToServerMapChunkTiles] MapChunkEntityUpdate ${update.key} " + "contained ${unboundedEntities.size} out-of-bounds entities " + unboundedEntities.joinToString(
+//          limit = 10
+//        )
+//      )
+//    }
+//
+//    // TODO There might be multiple resources per tile position, therefore this will arbitrarily
+//    //      exclude resources. Maybe change it from a Map to a List? Or have multiple resources per
+//    //      tile position?
+//    val mapValidEntitiesToPrototypeHash = validEntityPositions.associate { (pos, resource) ->
+//      pos to PrototypeHashCode(resource.protoId)
+//    }
+
+    val entitiesToPrototypeHash = update.entities().associate { entity ->
+      entity.position.toMapTilePosition() to PrototypeHashCode(entity.protoId)
+    }
+
+    val chunkId = ServerMapChunkId(
+      serverId = serverId,
+      layer = ServerMapTileLayer.BUILDING,
+      chunkPosition = update.key.chunkPosition,
+      surfaceIndex = update.key.surfaceIndex,
+    )
+
+    val tiles = ServerMapChunkTiles(chunkId, entitiesToPrototypeHash)
+
+    reChunkTiles(pid, tiles)
+  }
+}
+
+
+fun KStream<FactorioServerId, MapChunkResourceUpdate>.convertResourceUpdateToServerMapChunkTiles(
+  callerPid: String,
+): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
+
+  val pid = "$callerPid.convertResourceUpdateToChunkTiles"
+
+  return flatMap("$pid.map") { serverId: FactorioServerId, update: MapChunkResourceUpdate ->
+
+//    // TODO There might be multiple resources per tile position, therefore this will arbitrarily
+//    //      exclude resources. Maybe change it from a Map to a List? Or have multiple resources per
+//    //      tile position?
+//    val mapValidEntitiesToPrototypeHash = validEntityPositions.associate { (pos, resource) ->
+//      pos to PrototypeHashCode(resource.protoId)
+//    }
+
+    val entitiesToPrototypeHash = update.resources().associate { entity ->
+      entity.position.toMapTilePosition() to PrototypeHashCode(entity.protoId)
+    }
+
+    val chunkId = ServerMapChunkId(
+      serverId = serverId,
+      layer = ServerMapTileLayer.RESOURCE,
+      chunkPosition = update.key.chunkPosition,
+      surfaceIndex = update.key.surfaceIndex,
+    )
+
+    val tiles = ServerMapChunkTiles(chunkId, entitiesToPrototypeHash)
+
+    reChunkTiles(pid, tiles)
+  }
+}
+
+
+//private fun KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.reduceServerMapResourceTilesToTable()
+//    : KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
+//
+//  val pid = "$pid.reduceServerMapResourceTilesToTable"
+//
+//  return repartition(
+//    repartitionedAs(
+//      "$pid.pre-table-repartition",
+//      kxsBinary.serde<ServerMapChunkId>(),
+//      kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
+//      // force, otherwise KTable-KTable FK join doesn't work
+//      numberOfPartitions = 1,
+//    )
+//  ).groupByKey(
+//    groupedAs(
+//      "$pid.group-by-key",
+//      kxsBinary.serde<ServerMapChunkId>(),
+//      kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
+//    )
+//  ).reduce(
+//    "$pid.reduce",
+//    materializedAs(
+//      "$pid.reduce.store",
+//      kxsBinary.serde<ServerMapChunkId>(),
+//      kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
+//    )
+//  ) { chunkTiles, otherChunkTiles ->
+//    chunkTiles + otherChunkTiles
+//  }
+//}
+
+
+private fun KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.enrichWithColourData(
+  callerPid: String,
+  protosTable: KTable<FactorioServerId, EntityPrototypeMap>,
+): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
+
+  val pid = "$callerPid.enrichWithColourData"
+
+  return join(
+    other = protosTable,
+    tableJoined = tableJoined("$pid.join-tiles-with-prototypes"),
+    materialized = materializedAs(
+      "$pid.join-tiles-with-prototypes.store",
+      kxsBinary.serde(),
+      kxsBinary.serde(),
+    ),
+    foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<PrototypeHashCode> ->
+      chunkTiles.chunkId.serverId
+    }) { tiles: ServerMapChunkTiles<PrototypeHashCode>, prototypes: EntityPrototypeMap ->
+    println("joining tiles:${tiles.map.size} with prototypes:${prototypes.size}")
+
+    val missingProtos = mutableSetOf<PrototypeHashCode>()
+
+    val tileColours: Map<MapTilePosition, ColourHex> = buildList {
+      tiles.map.forEach { (position, code) ->
+        when (val proto = prototypes[code]) {
+
+          null -> missingProtos += code
+
+          else -> when (val box = proto.collisionBox) {
+
+            null -> add(position to proto.mapColour())
+
+            else -> box.iterator(position).forEach { entitySubTile ->
+              add(entitySubTile to proto.mapColour())
+            }
+          }
+        }
+      }
+    }.fold(mapOf()) { acc, (pos, colour) ->
+      // merge overlapping colours
+      val newColour = when (val existing = acc[pos]) {
+        null -> colour
+        else -> (existing / 2u) + (colour / 2u)
+      }
+      acc + (pos to newColour)
+    }
+
+    if (missingProtos.isNotEmpty()) {
+      println("[$pid] missing ${missingProtos.size} tile prototypes: ${missingProtos.joinToString { "${it.code}" }}")
+    }
+
+    println("[$pid] Set ${tileColours.size} tile colours for chunk ${tiles.chunkId}")
+
+    ServerMapChunkTiles(tiles.chunkId, tileColours)
+  }
+}
+
+
+private fun MapBoundingBox.iterator(origin: MapTilePosition): Iterator<MapTilePosition> {
+
+  val xOffset: Int = floor(tileWidth.toDouble() / 2).toInt()
+  val startX = origin.x - xOffset
+  val endX = origin.x + xOffset
+
+  val yOffset: Int = floor(tileHeight.toDouble() / 2).toInt()
+  val startY = origin.y - yOffset
+  val endY = origin.y + yOffset
+
+  return iterator {
+    for (x in (startX..endX)) {
+      for (y in (startY..endY)) { // screen coordinates = +y points to the floor, -y to the ceiling
+        yield(MapTilePosition(x, y))
+      }
+    }
+  }
+}
+
+
+/**
+ * Rounds a number 'up', away from zero.
+ * (Towards positive infinity if positive, and towards negative infinity if negative.)
+ */
+private fun Double.roundToInfinity(): Int = when {
+  this > 0 -> ceil(this).toInt()
+  this < 0 -> floor(this).toInt()
+  else     -> 0
+}
+// ceil(abs(this)).toInt() * this.sign.toInt()
+
+
+private fun FactorioPrototype.Entity.mapColour(): ColourHex =
+  (colour ?: mapColour ?: mapColourFriend)?.toHex() ?: ColourHex.WHITE
+
+
+private operator fun ColourHex.div(divisor: UInt): ColourHex = copy(
+  red = (red / divisor).toUByte(),
+  green = (green / divisor).toUByte(),
+  blue = (blue / divisor).toUByte(),
+  alpha = (alpha / divisor).toUByte(),
+)
+
+
+private operator fun ColourHex.plus(other: ColourHex): ColourHex = copy(
+  red = (red + other.red).toUByte(),
+  green = (green + other.green).toUByte(),
+  blue = (blue + other.blue).toUByte(),
+  alpha = (alpha + other.alpha).toUByte(),
+)
+
+
+//// https://stackoverflow.com/a/9355778/4161471
+//private operator fun ColourHex.plus(other: ColourHex): ColourHex {
+//
+//  @Suppress("LocalVariableName")
+//  val `255` = UByte.MAX_VALUE
+//
+//  val compositeAlpha: UByte = (
+//      `255` - ((`255` - other.alpha) * (`255` - this.alpha)) / `255`
+//      ).toUByte()
+//
+//  if (compositeAlpha == UByte.MIN_VALUE) {
+//    return ColourHex.TRANSPARENT
+//  } else {
+//
+//    fun compositeColour(
+//      colour: (ColourHex) -> UByte,
+//    ): UByte {
+//      val colour1 = colour(this)
+//      val colour2 = colour(other)
+//      return ((
+//          (
+//              ( colour2 * other.alpha) + (colour1 * this.alpha * ( other.alpha))
+//              ) / compositeAlpha
+//          ) / `255`
+//          ).toUByte()
+//    }
+//
+//    return ColourHex(
+//      compositeColour(ColourHex::red),
+//      compositeColour(ColourHex::green),
+//      compositeColour(ColourHex::blue),
+//      compositeAlpha,
+//    )
+//  }
+//}
+
+//private fun KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.streamMapChunkColouredTo(
+//  chunkSize: ChunkSize,
+//  outputTopic: String,
+//) {
+//  val pid = "$pid.output-chunk.${chunkSize.name}"
+//
+//  toStream("$pid.stream")
+//    .filter("$pid.filter-tiles-not-empty") { _, chunkTiles ->
+//      !chunkTiles?.map.isNullOrEmpty()
+//    }
+//    .mapValues("$pid.map-not-null") { _, chunkTiles ->
+//      requireNotNull(chunkTiles)
+//    }
+//    .peek("$pid.print-group-result") { _, chunkTiles ->
+//      println("Grouping map tiles result: ${chunkTiles.chunkId} / size:${chunkTiles.map.size}")
+//    }.to(
+//      outputTopic,
+//      producedAs(
+//        "$pid.grouped-map-chunks",
+//        kxsBinary.serde<ServerMapChunkId>(),
+//        kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
+//      )
+//    )
+//}
+
+
+///**
+// * Group [MapChunkTileUpdate]s by the Chunk position.
+// *
+// * They should already be grouped, but do it again to make sure, and to filter out empty updates.
+// */
+//private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.groupByChunkPosition(
+//  chunkSize: ChunkSize = ChunkSize.CHUNK_032
+//): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
+//
+//  val pid = "$pid.groupByChunkPosition.${chunkSize.name}"
+//
+//  return map("$pid.change-chunk-size") { chunkId: ServerMapChunkId, chunkTiles: ServerMapChunkTiles<ColourHex>? ->
+//
+//    val newChunkPosition = chunkId.chunkPosition
+//      .leftTopTile(chunkId.chunkSize)
+//      .toMapChunkPosition(chunkSize)
+//
+//    val newId = chunkId.copy(
+//      chunkSize = chunkSize,
+//      chunkPosition = newChunkPosition,
+//    )
+//
+//    val newChunkTiles = chunkTiles?.map ?: emptyMap()
+//
+//    newId to ServerMapChunkTiles(chunkId = newId, map = newChunkTiles)
+//  }.groupByKey(
+//    groupedAs(
+//      "$pid.group-by-key",
+//      kxsBinary.serde<ServerMapChunkId>(),
+//      kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
+//    )
+//  ).reduce(
+//    "$pid.reduce",
+//    materializedAs(
+//      "$pid.reduce.store",
+//      kxsBinary.serde<ServerMapChunkId>(),
+//      kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
+//    )
+//  ) { chunkTiles, otherChunkTiles ->
+//    chunkTiles + otherChunkTiles
+//  }
+//}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/factorioServerPackets.kt

@@ -3,7 +3,7 @@ package dev.adamko.kafkatorio.server.processor.topology
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
 import dev.adamko.kafkatorio.schema.common.Tick
 import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
-import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
+import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
 import dev.adamko.kafkatorio.server.config.jsonMapper
 import dev.adamko.kafkatorio.server.processor.TOPIC_SRC_SERVER_LOG
 import dev.adamko.kafkatorio.server.processor.topicName
@@ -38,20 +38,18 @@ fun factorioServerPacketStream(
       KafkatorioPacket(
         modVersion = "unknown",
         tick = Tick(0u),
-        data = KafkatorioPacketData.Error(
+        data = KafkatorioPacketDataError(
           message = e.message,
           rawValue = value,
         )
       )
     }
 
-    if (packet.data is KafkatorioPacketData.Error) {
+    if (packet.data is KafkatorioPacketDataError) {
       println("error parsing $TOPIC_SRC_SERVER_LOG message: ${packet.data}")
     }
 
     FactorioServerId(serverId) to packet
-  }.filter { _, value ->
-    value.data !is KafkatorioPacketData.Error
   }
 }
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/model.kt

@@ -10,7 +10,7 @@ import kotlinx.serialization.Serializable
 
 
 @Serializable
-data class ServerMapChunkTiles<Data>(
+data class ServerMapChunkTiles<Data : Any>(
   val chunkId: ServerMapChunkId,
   val map: Map<MapTilePosition, Data>,
 ) {
@@ -19,7 +19,7 @@ data class ServerMapChunkTiles<Data>(
 
 
 val MapTile.position
-  get() = MapTilePosition(x, y)
+  get() = MapTilePosition(x = x, y = y)
 
 
 val TRANSPARENT_AWT: Color = ColourHex.TRANSPARENT.toRgbColor().awt()

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/prototypes.kt

@@ -2,36 +2,34 @@ package dev.adamko.kafkatorio.server.processor.topology
 
 import dev.adamko.kafkatorio.library.kxsBinary
 import dev.adamko.kafkatorio.schema.common.ColourHex
+import dev.adamko.kafkatorio.schema.common.FactorioEntityData
 import dev.adamko.kafkatorio.schema.common.FactorioPrototype
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
-import dev.adamko.kafkatorio.schema.common.MapTile
+import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
+import dev.adamko.kafkatorio.schema.common.PrototypeId
 import dev.adamko.kafkatorio.schema.common.toHex
 import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
+import dev.adamko.kotka.extensions.groupedAs
 import dev.adamko.kotka.extensions.materializedAs
-import dev.adamko.kotka.extensions.repartitionedAs
-import dev.adamko.kotka.extensions.streams.filter
+import dev.adamko.kotka.extensions.materializedWith
+import dev.adamko.kotka.extensions.*
+import dev.adamko.kotka.extensions.streams.*
 import dev.adamko.kotka.extensions.streams.mapValues
 import dev.adamko.kotka.extensions.streams.toTable
+import dev.adamko.kotka.extensions.tables.*
 import dev.adamko.kotka.kxs.serde
 import kotlinx.serialization.Serializable
 import org.apache.kafka.streams.kstream.KStream
 import org.apache.kafka.streams.kstream.KTable
 
 
-/** Use the hashcode of Prototypes names as keys - more efficient storage */
-@JvmInline
-@Serializable
-value class TileProtoHashCode private constructor(val code: Int) {
-  constructor(tile: MapTile) : this(tile.protoId.hashCode())
-  constructor(tile: FactorioPrototype.MapTile) : this(tile.protoId.hashCode())
-}
-
-
 @JvmInline
 @Serializable
 value class TileColourDict(
-  private val map: Map<TileProtoHashCode, ColourHex>
-) : Map<TileProtoHashCode, ColourHex> by map
+  private val map: Map<PrototypeHashCode, ColourHex>
+) : Map<PrototypeHashCode, ColourHex> by map {
+  operator fun plus(other: TileColourDict): TileColourDict = TileColourDict(this.map + other.map)
+}
 
 
 /** Get the latest map-tile colours per server. */
@@ -46,7 +44,7 @@ fun KStream<FactorioServerId, PrototypesUpdate>.createTilePrototypeTable()
     val map = protoPacket
       .prototypes
       .filterIsInstance<FactorioPrototype.MapTile>()
-      .associate { TileProtoHashCode(it) to it.mapColour.toHex() }
+      .associate { PrototypeHashCode(it) to it.mapColour.toHex() }
 
     TileColourDict(map)
   }.filter("$pid.filterMapTileProtos") { _, dict: TileColourDict ->
@@ -70,3 +68,71 @@ fun KStream<FactorioServerId, PrototypesUpdate>.createTilePrototypeTable()
     )
   )
 }
+
+
+@JvmInline
+@Serializable
+value class EntityPrototypeMap(
+  private val map: Map<PrototypeHashCode, FactorioPrototype.Entity>
+) : Map<PrototypeHashCode, FactorioPrototype.Entity> by map {
+  operator fun plus(other: EntityPrototypeMap): EntityPrototypeMap =
+    EntityPrototypeMap(this.map + other.map)
+
+  operator fun get(protoId: PrototypeId): FactorioPrototype.Entity? =
+    map[PrototypeHashCode(protoId)]
+
+  operator fun get(entity: FactorioEntityData): FactorioPrototype.Entity? =
+    map[PrototypeHashCode(entity.protoId)]
+}
+
+
+/** Get the latest entity colours per server. */
+fun KStream<FactorioServerId, PrototypesUpdate>.createEntityPrototypeMap(
+  callerPid: String
+): KTable<FactorioServerId, EntityPrototypeMap> {
+
+  val pid = "$callerPid.createEntityPrototypeMap"
+
+  return mapValues("$pid.colour-dict.build") { _, protoPacket: PrototypesUpdate ->
+    val map = protoPacket
+      .prototypes
+      .filterIsInstance<FactorioPrototype.Entity>()
+      .associateBy { entityProto ->
+        PrototypeHashCode(entityProto)
+      }
+
+    EntityPrototypeMap(map)
+  }.filter("$pid.filterEntityProtos") { _, dict: EntityPrototypeMap ->
+    dict.isNotEmpty()
+  }.groupByKey(groupedAs("$pid.group-by-key"))
+    .reduce(
+      "$pid.merge",
+      materializedWith(
+        kxsBinary.serde(),
+        kxsBinary.serde(),
+      )
+    ) { dict1: EntityPrototypeMap, dict2: EntityPrototypeMap ->
+      dict1 + dict2
+    }.toStream("$pid.")
+    .filter("$pid.not-null.filter") { _, v -> v != null }
+    .mapValues("$pid.not-null.require") { _, value ->
+      requireNotNull(value)
+    }.peek { serverId, dict: EntityPrototypeMap ->
+      println("$pid: server $serverId has EntityPrototypeMap (size:${dict.size}): ${dict.entries.joinToString()}")
+    }.repartition(
+      repartitionedAs(
+        "$pid.pre-table-repartition",
+        kxsBinary.serde(),
+        kxsBinary.serde(),
+        // force, otherwise KTable-KTable FK join doesn't work
+        numberOfPartitions = 1,
+      )
+    ).toTable(
+      "$pid.create-table",
+      materializedAs(
+        "$pid.output-store",
+        kxsBinary.serde(),
+        kxsBinary.serde(),
+      )
+    )
+}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/saveMapTiles.kt

@@ -8,11 +8,13 @@ import dev.adamko.kafkatorio.schema.common.ChunkSize
 import dev.adamko.kafkatorio.schema.common.ColourHex
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
 import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
-import dev.adamko.kafkatorio.schema.common.TilePngFilename
+import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
 import dev.adamko.kafkatorio.schema.packets.EventServerPacket
 import dev.adamko.kafkatorio.server.config.jsonMapper
 import dev.adamko.kafkatorio.server.processor.TOPIC_BROADCAST_TO_WEBSOCKET
-import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_COLOURED_STATE
+import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE
+import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE
+import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
 import dev.adamko.kotka.extensions.consumedAs
 import dev.adamko.kotka.extensions.producedAs
 import dev.adamko.kotka.extensions.stream
@@ -51,12 +53,14 @@ fun saveMapTiles(
         kxsBinary.serde(ServerMapChunkId.serializer()),
         kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
       ),
-      TOPIC_MAP_CHUNK_COLOURED_STATE,
+      TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
+      TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
+      TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
     ).filter("$pid.consume.grouped-map-chunks.filter-size") { _, y ->
       y.chunkId.chunkSize.lengthInTiles >= ChunkSize.CHUNK_256.lengthInTiles
     }
 
-  val savedChunkFilenames: KStream<ServerMapChunkId, TilePngFilename> =
+  val savedChunkFilenames: KStream<ServerMapChunkId, ServerMapTilePngFilename> =
     subdividedMapChunkTilesDebounced.saveChunkAsImage(serverDataDir)
 
   savedChunkFilenames.broadcastSavedTileUpdates()
@@ -67,7 +71,7 @@ fun saveMapTiles(
 
 private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.saveChunkAsImage(
   tileDirectory: Path,
-): KStream<ServerMapChunkId, TilePngFilename> = mapValues(
+): KStream<ServerMapChunkId, ServerMapTilePngFilename> = mapValues(
   "$pid.save-tiles"
 ) { chunkId: ServerMapChunkId, tileColours: ServerMapChunkTiles<ColourHex> ->
 
@@ -81,19 +85,36 @@ private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.saveChunkA
   val chunkOriginX: Int = chunkId.chunkPosition.x * chunkId.chunkSize.lengthInTiles
   val chunkOriginY: Int = chunkId.chunkPosition.y * chunkId.chunkSize.lengthInTiles
 
+  val validPixelRangeX = 0 until chunkImage.width
+  val validPixelRangeY = 0 until chunkImage.height
+
   tileColours.map.forEach { (tilePosition, colour) ->
     val rgbColour = colour.toRgbColor()
 
     val pixelX = abs(abs(tilePosition.x) - abs(chunkOriginX))
     val pixelY = abs(abs(tilePosition.y) - abs(chunkOriginY))
 
+    if (!(pixelX in validPixelRangeX && pixelY in validPixelRangeY)) {
+      // TODO figure out why buildings on chunk boundaries aren't re-chunked, and so cause this error
+      println(
+        "ERROR - pixel XY:[$pixelX,$pixelY] was out of bounds (x:$validPixelRangeX, y:$validPixelRangeY). $tilePosition, $colour, chunkOriginXY[$chunkOriginX,$chunkOriginY], $chunkId"
+      )
+    } else {
+      try {
         chunkImage.setColor(pixelX, pixelY, rgbColour)
+      } catch (e: Exception) {
+        println(
+          "ERROR $e - pixel XY:[$pixelX,$pixelY] (x:$validPixelRangeX, y:$validPixelRangeY). $tilePosition, $colour, chunkOriginXY[$chunkOriginX,$chunkOriginY], $chunkId"
+        )
+        throw e
+      }
+    }
   }
 
   val scaledImage =
     chunkImage.scaleTo(WEB_MAP_TILE_SIZE_PX, WEB_MAP_TILE_SIZE_PX, ScaleMethod.FastScale)
 
-  val filename = TilePngFilename(chunkId)
+  val filename = ServerMapTilePngFilename(chunkId)
   saveTile(filename, tileDirectory, scaledImage)
   filename
 }
@@ -101,7 +122,7 @@ private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.saveChunkA
 
 @Synchronized
 private fun saveTile(
-  filename: TilePngFilename,
+  filename: ServerMapTilePngFilename,
   serverDataDir: Path,
   img: ImmutableImage,
 ) {
@@ -124,8 +145,8 @@ private fun saveTile(
 }
 
 
-private fun KStream<ServerMapChunkId, TilePngFilename>.broadcastSavedTileUpdates() {
-  map<ServerMapChunkId, TilePngFilename, FactorioServerId, EventServerPacket>(
+private fun KStream<ServerMapChunkId, ServerMapTilePngFilename>.broadcastSavedTileUpdates() {
+  map<ServerMapChunkId, ServerMapTilePngFilename, FactorioServerId, EventServerPacket>(
     "$pid.broadcast.filename-update.convert",
   ) { id, filename ->
     id.serverId to EventServerPacket.ChunkTileSaved(id, filename)

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/socket/SyslogSocketServer.kt

@@ -69,7 +69,7 @@ class SyslogSocketServer(
                     log("received 'null' from ${socket.description()} - closing connection")
                     socket.close()
                   }
-                  else -> log("received '$line' from ${socket.description()}")
+//                  else -> log("received '$line' from ${socket.description()}")
                 }
                 line
               },

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/factorioServersRoutes.kt

@@ -2,8 +2,9 @@ package dev.adamko.kafkatorio.server.web.rest
 
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
 import dev.adamko.kafkatorio.schema.common.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
+import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
 import dev.adamko.kafkatorio.schema.common.SurfaceIndex
-import dev.adamko.kafkatorio.schema.common.TilePngFilename
 import dev.adamko.kafkatorio.server.config.ApplicationProperties
 import io.ktor.http.HttpStatusCode
 import io.ktor.http.content.EntityTagVersion
@@ -41,8 +42,9 @@ private suspend fun PipelineContext<Unit, ApplicationCall>.handleTileRequest(
   appProps: ApplicationProperties,
 ) {
 
-  val tileFilename = TilePngFilename(
+  val tileFilename = ServerMapTilePngFilename(
     tile.serverId,
+    tile.layer,
     tile.surfaceIndex,
     tile.zoomLevel,
     tile.chunkX,
@@ -60,6 +62,19 @@ private suspend fun PipelineContext<Unit, ApplicationCall>.handleTileRequest(
     }
   } else {
     call.respond(HttpStatusCode.NotFound)
+//    // It's not a client error to request this tile, it just doesn't exist (yet). So return 304?
+//    call.response.cacheControl(CacheControl.NoStore(null))
+//
+//    HeadersBuilder(size = 2).apply {
+//      EntityTagVersion(System.currentTimeMillis().hashCode().toString())
+//        .appendHeadersTo(this)
+//      LastModifiedVersion(System.currentTimeMillis())
+//        .appendHeadersTo(this)
+//    }.build().flattenForEach { name, value ->
+//      call.response.header(name, value)
+//    }
+//
+//    call.respond(HttpStatusCode.NotModified)
   }
 }
 
@@ -82,9 +97,11 @@ data class FactorioServer(
   ) {
 
     @Serializable
-    @Resource("tiles/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png")
+    @Resource("layers/{layer}/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png")
     data class Tile(
       val mapData: MapData,
+
+      val layer: ServerMapTileLayer,
       val surfaceIndex: SurfaceIndex,
       val zoomLevel: Int,
       val chunkX: Int,

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/restApi.kt

@@ -3,6 +3,7 @@ package dev.adamko.kafkatorio.server.web.rest
 import dev.adamko.kafkatorio.server.config.ApplicationProperties
 import io.ktor.http.ContentType
 import io.ktor.http.HttpStatusCode
+import io.ktor.http.isSuccess
 import io.ktor.serialization.kotlinx.json.json
 import io.ktor.server.application.Application
 import io.ktor.server.application.install
@@ -24,7 +25,12 @@ import io.ktor.server.routing.routing
 fun Application.webmapTileServer(appProps: ApplicationProperties) {
 
   install(AutoHeadResponse)
-  install(CallLogging)
+  install(CallLogging) {
+    filter { call ->
+      // only log failures
+      !(call.response.status()?.isSuccess() ?: false)
+    }
+  }
   install(Compression) {
     default()
     minimumSize(1024)

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/WebmapWebsocketServer.kt

@@ -97,7 +97,9 @@ class WebmapWebsocketServer {
     override suspend fun send(frame: Frame): Unit = runCatching {
       super.send(frame)
     }.fold(
-      onSuccess = { log("sent message to client $name: $frame") },
+      onSuccess = {
+//        log("sent message to client $name: $frame")
+      },
       onFailure = { e -> log("failed to send message to client $name: $e") }
     )
 

modules/events-server/src/main/resources/config.yml

@@ -5,7 +5,7 @@ server-data-dir: ${SERVER_DATA_DIR:-./server-data}
 
 
 socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}
-socket-server-port: ${SOCKET_SERVER_PORT:-12092}
+#socket-server-port: ${SOCKET_SERVER_PORT:-12092}
 
 
 kafka-admin-config:

modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/serdeSize.kt

@@ -7,6 +7,7 @@ import dev.adamko.kafkatorio.schema.common.ColourHex
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
 import dev.adamko.kafkatorio.schema.common.MapChunkPosition
 import dev.adamko.kafkatorio.schema.common.MapTilePosition
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
 import dev.adamko.kafkatorio.schema.common.SurfaceIndex
 import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
 import java.util.UUID
@@ -22,9 +23,9 @@ class Blah {
     val tiles = ServerMapChunkTiles<ColourHex>(
       chunkId = ServerMapChunkId(
         serverId = FactorioServerId(UUID.randomUUID().toString()),
-        chunkPosition = MapChunkPosition(Int.MAX_VALUE, Int.MAX_VALUE),
+        layer = ServerMapTileLayer.BUILDING,
+        chunkPosition = MapChunkPosition(Int.MAX_VALUE, Int.MAX_VALUE, ChunkSize.MAX),
         surfaceIndex = SurfaceIndex(UInt.MAX_VALUE),
-        chunkSize = ChunkSize.MAX,
       ),
       buildMap {
         repeat(512) { x ->

modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/server/processor/topology/SaveMapChunksTest.kt

@@ -1,15 +1,16 @@
 package dev.adamko.kafkatorio.server.processor.topology
 
+import dev.adamko.kafkatorio.library.iterateTiles
 import dev.adamko.kafkatorio.library.kxsBinary
+import dev.adamko.kafkatorio.library.toMapChunkPosition
 import dev.adamko.kafkatorio.schema.common.ChunkSize
 import dev.adamko.kafkatorio.schema.common.ColourHex
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
 import dev.adamko.kafkatorio.schema.common.MapChunkPosition
 import dev.adamko.kafkatorio.schema.common.MapTilePosition
 import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
 import dev.adamko.kafkatorio.schema.common.SurfaceIndex
-import dev.adamko.kafkatorio.schema.common.iterateTiles
-import dev.adamko.kafkatorio.schema.common.toMapChunkPosition
 import dev.adamko.kotka.kxs.serde
 import io.kotest.core.spec.style.FunSpec
 import io.kotest.property.Arb
@@ -55,11 +56,11 @@ class SaveMapChunksTest : FunSpec({
         scenario += 1.minutes
         delay(5.seconds)
         scenario += 1.minutes
-//        delay(5.seconds)
+        // delay(5.seconds)
         scenario += 1.minutes
-//        delay(5.seconds)
+        // delay(5.seconds)
         scenario += 1.minutes
-//        delay(5.seconds)
+        // delay(5.seconds)
 
         println("file:///${scenario.outputDir.absolutePathString()}")
       }
@@ -71,7 +72,7 @@ class SaveMapChunksTest : FunSpec({
   class Scenario(
     streamsBuilder: StreamsBuilder = StreamsBuilder(),
 
-    groupedMapChunksInputTopicName: String = "kafkatorio.state.map-chunk.colour",
+    groupedMapChunksInputTopicName: String = "kafkatorio.state.map-chunk.terrain.colour",
 
     val serverId: FactorioServerId = FactorioServerId("test-server-id"),
   ) : AutoCloseable {
@@ -104,8 +105,9 @@ class SaveMapChunksTest : FunSpec({
     private fun mapChunkPositionArb(
       xArb: Arb<Int> = Arb.int(-10..10),
       yArb: Arb<Int> = Arb.int(-10..10),
+      chunkSize: ChunkSize = ChunkSize.MAX,
     ): Arb<MapChunkPosition> = arbitrary {
-      MapChunkPosition(xArb.bind(), yArb.bind())
+      MapChunkPosition(xArb.bind(), yArb.bind(), chunkSize)
     }
 
     private fun colourHexArb(
@@ -124,15 +126,15 @@ class SaveMapChunksTest : FunSpec({
 
     private fun serverMapChunkIdArb(
       serverId: FactorioServerId,
-      chunkPosition: Arb<MapChunkPosition> = mapChunkPositionArb(),
-      surfaceIndex: SurfaceIndex = SurfaceIndex(1u),
       chunkSize: ChunkSize = ChunkSize.MAX,
+      chunkPosition: Arb<MapChunkPosition> = mapChunkPositionArb(chunkSize = chunkSize),
+      surfaceIndex: SurfaceIndex = SurfaceIndex(1u),
     ): Arb<ServerMapChunkId> = arbitrary {
       ServerMapChunkId(
         serverId = serverId,
+        layer = ServerMapTileLayer.RESOURCE,
         chunkPosition = chunkPosition.bind(),
         surfaceIndex = surfaceIndex,
-        chunkSize = chunkSize,
       )
     }
 
@@ -145,7 +147,7 @@ class SaveMapChunksTest : FunSpec({
       println("chunk size: ${chunkId.chunkSize} ${chunkId.chunkSize.lengthInTiles}")
 
       val map = chunkId.chunkPosition
-        .iterateTiles(chunkId.chunkSize)
+        .iterateTiles()
         .asSequence()
         .map { tilePos ->
           val colour =
@@ -154,7 +156,7 @@ class SaveMapChunksTest : FunSpec({
             }
           tilePos to colour
         }
-//        .take(2500)
+//        .take(100)
         .toMap()
 
       println(map.size)

modules/infra-kafka-cluster/src/docker-compose.yml

@@ -14,6 +14,7 @@ services:
         KAFKA_DL_URL: "https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_2.13-${KAFKA_VERSION}.tgz"
     volumes:
       - "./kafka-server.properties:/kafka/server.properties"
+#      - "./kafka-data:/tmp/kraft-combined-logs" # see Dockerfile
     environment:
       JMX_PORT: "9991"
       #      KAFKA_JMX_OPTS: "9991"

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/App.kt

@@ -78,7 +78,7 @@ class App(
     wsService.packetsFlow
       .filterIsInstance<EventServerPacket.ChunkTileSaved>()
       .onEach { tileSavedEvent ->
-        println("[packetsFlow] triggering tile refresh ${tileSavedEvent.filename.value}")
+//        println("[packetsFlow] triggering tile refresh ${tileSavedEvent.filename.value}")
         gameState.map.refreshUpdatedTilePng(tileSavedEvent.filename)
       }.launchIn(this)
 

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt

@@ -1,6 +1,7 @@
 package dev.adamko.kafkatorio.webmap
 
-import dev.adamko.kafkatorio.schema.common.TilePngFilename
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
+import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
 import dev.adamko.kafkatorio.webmap.externals.TileOnLoadFn
 import dev.adamko.kafkatorio.webmap.externals.tileOnLoad
 import io.kvision.maps.Maps
@@ -30,9 +31,7 @@ import org.w3c.fetch.RequestInit
 import org.w3c.fetch.RequestMode
 
 
-class FactorioMap(
-  private val tileUrlTemplate: String = """/kafkatorio/data/servers/syslog-test/map/tiles/s1/z{z}/x{x}/y{y}.png""",
-) {
+class FactorioMap {
 
   val kvMap: Maps = Maps {
     width = 800.px
@@ -40,18 +39,22 @@ class FactorioMap(
     margin = 10.px
   }
 
-  private val factorioTerrainLayer: TileLayer<*> = buildFactorioTerrainLayer()
+  private val terrainTileLayer: TileLayer<*> = createServerTileLayer(ServerMapTileLayer.TERRAIN)
+  private val resourcesTileLayer: TileLayer<*> = createServerTileLayer(ServerMapTileLayer.RESOURCE)
+  private val buildingsTileLayer: TileLayer<*> = createServerTileLayer(ServerMapTileLayer.BUILDING)
 
   val playerIconsLayer: LayerGroup = LayerGroup()
 
   init {
 
     val baseLayers: Control.LayersObject = jso {
-      set("Terrain", factorioTerrainLayer)
+      set("Terrain", terrainTileLayer)
     }
 
     val overlays: Control.LayersObject = jso {
       set("Players", playerIconsLayer)
+      set("Resources", resourcesTileLayer)
+      set("Buildings", buildingsTileLayer)
     }
 
     val layersControl = Layers(
@@ -76,14 +79,17 @@ class FactorioMap(
 //    options.fadeAnimation = false
       addControl(layersControl)
 
-      factorioTerrainLayer.addTo(this)
+      terrainTileLayer.addTo(this)
+      resourcesTileLayer.addTo(this)
+      buildingsTileLayer.addTo(this)
+
       playerIconsLayer.addTo(this)
     }
   }
 
 
   suspend fun refreshUpdatedTilePng(
-    tilePngFilename: TilePngFilename,
+    tilePngFilename: ServerMapTilePngFilename,
   ): Unit = coroutineScope {
     document
       .querySelectorAll("img.leaflet-tile-loaded")
@@ -111,10 +117,14 @@ class FactorioMap(
       }
   }
 
-  private fun buildFactorioTerrainLayer(): TileLayer<TileLayer.TileLayerOptions> {
+  private fun createServerTileLayer(
+    layer: ServerMapTileLayer
+  ): TileLayer<TileLayer.TileLayerOptions> {
+
+    val urlTemplate: String = tileLayerUrlTemplate(layer)
 
     val baseTileLayer = L.tileLayer(
-      tileUrlTemplate
+      urlTemplate
     ) {
       attribution = "kafkatorio"
       tileSize = 256
@@ -147,5 +157,8 @@ class FactorioMap(
 
   companion object {
     const val DYNAMIC_RELOAD_ATT = "dynamic-reload"
+
+    private fun tileLayerUrlTemplate(layer: ServerMapTileLayer): String =
+      "/kafkatorio/data/servers/syslog-test/map/layers/${layer.dir}/s1/z{z}/x{x}/y{y}.png"
   }
 }

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt

@@ -5,8 +5,9 @@ import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate
 import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate
 import dev.adamko.kafkatorio.schema.packets.EntityUpdate
 import dev.adamko.kafkatorio.schema.packets.EventServerPacket
-import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
+import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
 import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
+import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
 import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
 import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
 import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
@@ -76,18 +77,19 @@ class WebsocketService(
 
           is EventServerPacket.Kafkatorio     ->
             when (val packetData = packet.packet.data) {
-              is PlayerUpdate               -> reduxStore.dispatch(
-                FactorioUpdate.Player(packet.packet.tick, packetData)
-              )
+              is PlayerUpdate              ->
+                reduxStore.dispatch(FactorioUpdate.Player(packet.packet.tick, packetData))
+
               is ConfigurationUpdate,
               is ConsoleChatUpdate,
               is ConsoleCommandUpdate,
               is EntityUpdate,
               is MapChunkEntityUpdate,
+              is MapChunkResourceUpdate,
               is MapChunkTileUpdate,
               is PrototypesUpdate,
               is SurfaceUpdate,
-              is KafkatorioPacketData.Error -> {
+              is KafkatorioPacketDataError -> {
                 // to be continued...
               }
             }