so close to working performantly.... just the tiles are transparent???

- bump some versions
- MapTile no longer has objectName field
- fix nulls/optional fields in FactorioConfigurationUpdate
- use bytes to encode rgba colours - waaaayyyy smaller
- increase Kafka record size to ~10mb

build.gradle.kts

@@ -6,7 +6,7 @@ plugins {
 }
 
 group = "dev.adamko.kafkatorio"
-version = "0.2.7"
+version = "0.2.8"
 
 val licenseFile: RegularFile by extra(layout.projectDirectory.file("LICENSE"))
 

gradle/libs.versions.toml

@@ -10,17 +10,17 @@ kotlin     = "1.6.10"
 
 ## libs ##
 camel-kafkaConnectors = "0.11.0"
-http4k                = "4.17.6.0"
+http4k                = "4.19.1.0"
 kafka                 = "3.1.0"
 kotlinx-serialization = "1.3.2"
-kvision               = "5.7.1"
+kvision               = "5.8.1"
 log4j2                = "2.14.1"
 qosLogback            = "1.2.5"
 slf4j                 = "1.7.30"
 
 ## test ##
 junit      = "5.8.2"
-kotest     = "5.0.1"
+kotest     = "5.1.0"
 
 ## infra ##
 factorio               = "1.1.53"
@@ -99,7 +99,7 @@ kotka         = { group = "com.github.blueanvil", name = "kotka",         versio
 ### test  ###
 #############
 
-mockk = { group = "io.mockk", name = "mockk", version = "1.12.0" }
+mockk = { group = "io.mockk", name = "mockk", version = "1.12.2" }
 
 ## junit ##
 junit-bom      = { group = "org.junit",          name = "junit-bom",                version.ref = "junit"  }

modules/events-mod/src/main/typescript/generated/kafkatorio-schema/schema.d.ts

@@ -43,7 +43,6 @@ interface MapChunkPosition {
 }
 
 interface MapTile {
-  objectName: string;
   position: MapTilePosition;
   prototypeName: string;
 }

modules/events-mod/src/main/typescript/main/events/converters.ts

@@ -98,8 +98,6 @@ export namespace Converters {
 
   export function convertTile(tile: LuaTile): MapTile {
     return {
-      objectName: tile.object_name,
-
       position: mapTilePosition(tile.position),
       prototypeName: tile.prototype.name,
     }

modules/events-mod/src/main/typescript/main/events/handlers.ts

@@ -92,8 +92,7 @@ export function handleTilesUpdate(
 
     let convertedTiles: MapTile[] = updatedTiles.map((it) => {
       return {
-        objectName: "LuaTile",
+        position: it.position, // TODO use converters for LuaTile
-        position: it.position,
         prototypeName: newTileType.name,
       }
     })

modules/events-mod/src/main/typescript/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

modules/events-mod/src/main/typescript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "private": true,
   "scripts": {
     "build": "tstl",

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/converters/colour.kt

@@ -23,7 +23,12 @@ fun Colour.toHexadecimal() = if (isPercentage()) {
     (alpha * 255).coerceIn(0f..255f),
   )
 } else {
-  this
+  Colour(
+    red.coerceIn(0f..255f),
+    green.coerceIn(0f..255f),
+    blue.coerceIn(0f..255f),
+    alpha.coerceIn(0f..255f),
+  )
 }
 
 fun Colour.toPercentage() = if (isPercentage()) {

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt

@@ -1,5 +1,6 @@
 package dev.adamko.kafkatorio.events.schema
 
+import dev.adamko.kafkatorio.events.schema.converters.toHexadecimal
 import kotlinx.serialization.EncodeDefault
 import kotlinx.serialization.Serializable
 
@@ -56,7 +57,7 @@ data class MapBoundingBox(
  * Red, green, blue and alpha values, all in range `[0, 1]` or all in range `[0, 255]` if any
  * value is > 1.
  *
- * All values here are optional. Colour channels default to 0, the alpha channel defaults to 1.
+ * All values here are optional. Colour channels default to `0`, the alpha channel defaults to `1`.
  */
 @Serializable
 data class Colour(
@@ -68,4 +69,39 @@ data class Colour(
   val blue: Float = 0f,
   @EncodeDefault
   val alpha: Float = 1f,
+) {
+
+  fun toHex() = toHexadecimal().let {
+    ColourHex(
+      it.red.toUInt().toUByte(),
+      it.green.toUInt().toUByte(),
+      it.blue.toUInt().toUByte(),
+      it.alpha.toUInt().toUByte(),
+    )
+  }
+}
+
+
+/** Size-efficient version of [Colour] */
+@Serializable
+data class ColourHex(
+  @EncodeDefault
+  val red: UByte = UByte.MIN_VALUE,
+  @EncodeDefault
+  val green: UByte = UByte.MIN_VALUE,
+  @EncodeDefault
+  val blue: UByte = UByte.MIN_VALUE,
+  @EncodeDefault
+  val alpha: UByte = UByte.MAX_VALUE,
+) {
+  companion object {
+    val TRANSPARENT = ColourHex(0u, 0u, 0u, 0u)
+  }
+}
+
+
+@Serializable
+data class MapTile(
+  val prototypeName: String,
+  val position: MapTilePosition,
 )

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioConfiguration.kt

@@ -16,13 +16,13 @@ data class FactorioConfigurationUpdate(
 
 @Serializable
 data class FactorioGameDataUpdate(
-  val oldVersion: String?,
+  val oldVersion: String? = null,
-  val newVersion: String?,
+  val newVersion: String? = null,
 )
 
 @Serializable
 data class FactorioModInfo(
   val modName: String,
-  val currentVersion: String?,
+  val currentVersion: String? = null,
-  val previousVersion: String?,
+  val previousVersion: String? = null,
 )

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioLuaObjects.kt

@@ -137,17 +137,6 @@ data class MapTiles(
   override val objectName = ObjectName.LuaTiles
 }
 
-@Serializable
-data class MapTile(
-  val prototypeName: String,
-  val position: MapTilePosition,
-)
-//  : FactorioObjectData()
-{
-  //  @EncodeDefault
-  val objectName: String = ""
-}
-
 @Serializable
 data class ConsoleChatMessage(
   val authorPlayerIndex: UInt?,

modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/FactorioEventTest.kt

@@ -203,7 +203,6 @@ class FactorioEventTest : FunSpec({
           "surfaceIndex": 1,
           "tiles": [
             {
-              "objectName": "LuaTile",
               "position": {
                 "y": 10,
                 "x": -65
@@ -211,23 +210,20 @@ class FactorioEventTest : FunSpec({
               "prototypeName": "refined-concrete"
             },
             {
-              "objectName": "LuaTile",
               "position": {
                 "y": 11,
                 "x": -65
               },
-              "prototypeName": "refined-concrete"
+              "prototypeName": "red-earth"
             },
             {
-              "objectName": "LuaTile",
               "position": {
                 "y": 10,
                 "x": -64
               },
-              "prototypeName": "refined-concrete"
+              "prototypeName": "blue-cheese"
             },
             {
-              "objectName": "LuaTile",
               "position": {
                 "y": 11,
                 "x": -64
@@ -256,11 +252,11 @@ class FactorioEventTest : FunSpec({
                 position = MapTilePosition(-65, 10)
               ),
               MapTile(
-                prototypeName = "refined-concrete",
+                prototypeName = "red-earth",
                 position = MapTilePosition(-65, 11)
               ),
               MapTile(
-                prototypeName = "refined-concrete",
+                prototypeName = "blue-cheese",
                 position = MapTilePosition(-64, 10)
               ),
               MapTile(
@@ -279,6 +275,60 @@ class FactorioEventTest : FunSpec({
         expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaTiles
       }
 
+      test("Then: expect encode equals json") {
+        val encoded = jsonMapperKafkatorio.encodeToString(actual)
+        encoded.shouldEqualJson(json)
+      }
+    }
+  }
+
+  context("Given: packet FactorioConfigurationUpdate") {
+    // language=JSON
+    val json = """
+               {
+                 "modVersion": "0.2.8",
+                 "packetType": "CONFIG",
+                 "allMods": [
+                   {
+                     "modName": "kafkatorio-events",
+                     "currentVersion": "0.2.8",
+                     "previousVersion": "0.2.7"
+                   },
+                   {
+                     "modName": "base",
+                     "currentVersion": "1.1.53"
+                   }
+                 ],
+                 "factorioData": {}
+               }
+               """.trimIndent()
+    context("When: decoded") {
+      val actual: FactorioConfigurationUpdate = jsonMapperKafkatorio.decodeFromString(json)
+
+      test("parse") {
+
+        val expected = FactorioConfigurationUpdate(
+          modVersion = "0.2.8",
+          factorioData = FactorioGameDataUpdate(null, null),
+          allMods = listOf(
+            FactorioModInfo(
+              modName = "kafkatorio-events",
+              currentVersion = "0.2.8",
+              previousVersion = "0.2.7"
+            ),
+            FactorioModInfo(
+              modName = "base",
+              currentVersion = "1.1.53",
+              previousVersion = null
+            ),
+          )
+        )
+
+        actual shouldBe expected
+
+        expected.packetType shouldBe KafkatorioPacket.PacketType.CONFIG
+      }
+
       test("Then: expect encode equals json") {
         val encoded = jsonMapperKafkatorio.encodeToString(actual)
         encoded.shouldEqualJson(json)

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt

@@ -1,16 +1,16 @@
 package dev.adamko.kafkatorio.processor
 
-import dev.adamko.kafkatorio.processor.topology.buildFactorioServerMap
 import dev.adamko.kafkatorio.processor.topology.factorioServerPacketStream
+import dev.adamko.kafkatorio.processor.topology.groupTilesIntoChunksWithColours
 import dev.adamko.kafkatorio.processor.topology.playerUpdatesToWsServer
 import dev.adamko.kafkatorio.processor.topology.saveMapTiles
 import dev.adamko.kafkatorio.processor.topology.splitFactorioServerPacketStream
+import dev.adamko.kafkatorio.processor.topology.tileProtoColourDictionary
 import java.time.Duration
 import org.apache.kafka.streams.KafkaStreams
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.TopologyDescription
 
-
 class KafkatorioTopology(
   private val websocketServer: WebsocketServer,
   private val builder: StreamsBuilder = StreamsBuilder(),
@@ -20,17 +20,27 @@ class KafkatorioTopology(
     const val sourceTopic = "factorio-server-log"
   }
 
+
   fun build() {
 
+
     val packets = factorioServerPacketStream(builder)
 
+
+
     splitFactorioServerPacketStream(packets)
 
     playerUpdatesToWsServer(websocketServer, builder)
 
-    val serverMapDataTable = buildFactorioServerMap(packets)
+//    val serverMapDataTable = buildFactorioServerMap(packets, tilePrototypesTable)
 
-    saveMapTiles(serverMapDataTable)
+    val tileProtoColourDict = tileProtoColourDictionary(packets)
+
+    val tilePrototypesTable = groupTilesIntoChunksWithColours(
+      packets,
+      tileProtoColourDict,
+    )
+    saveMapTiles(tilePrototypesTable)
 
 
     // the old way...

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioGameState.kt

@@ -1,127 +1,196 @@
 package dev.adamko.kafkatorio.processor.topology
 
-import dev.adamko.kafkatorio.events.schema.FactorioConfigurationUpdate
+import dev.adamko.kafkatorio.events.schema.ColourHex
 import dev.adamko.kafkatorio.events.schema.FactorioEvent
-import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
 import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
 import dev.adamko.kafkatorio.events.schema.MapChunk
+import dev.adamko.kafkatorio.events.schema.MapChunkPosition
 import dev.adamko.kafkatorio.events.schema.MapTilePosition
-import dev.adamko.kafkatorio.events.schema.MapTilePrototype
 import dev.adamko.kafkatorio.events.schema.MapTiles
-import dev.adamko.kafkatorio.processor.serdes.jsonMapper
+import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
 import dev.adamko.kotka.extensions.groupedAs
 import dev.adamko.kotka.extensions.materializedAs
 import dev.adamko.kotka.extensions.materializedWith
-import dev.adamko.kotka.extensions.streams.aggregate
+import dev.adamko.kotka.extensions.streams.flatMap
-import dev.adamko.kotka.extensions.tables.mapValues
+import dev.adamko.kotka.extensions.streams.reduce
+import dev.adamko.kotka.extensions.tables.join
 import dev.adamko.kotka.kxs.serde
 import kotlinx.serialization.Serializable
 import org.apache.kafka.streams.kstream.KStream
 import org.apache.kafka.streams.kstream.KTable
 
 
+//@Serializable
+//data class FactorioServerMap(
+//  val serverId: FactorioServerId,
+//  val tilePrototypes: Map<PrototypeName, MapTilePrototype>,
+//  val surfaces: Map<SurfaceIndex, FactorioMapSurface>,
+//)
+
+
+//@Serializable
+//data class FactorioMapSurface(
+//  val index: SurfaceIndex,
+//  val tiles: Map<MapTilePosition, PrototypeName>,
+//)
+
+
 @Serializable
-data class FactorioServerMap(
+data class ServerMapChunkId(
   val serverId: FactorioServerId,
-  val tilePrototypes: Map<PrototypeName, MapTilePrototype>,
+  val chunkPosition: MapChunkPosition,
-  val surfaces: Map<SurfaceIndex, FactorioMapSurface>,
+  val surfaceIndex: SurfaceIndex,
+  val zoomLevel: ZoomLevel,
 )
 
 
 @Serializable
-data class FactorioMapSurface(
+data class ServerMapChunkTiles<Data>(
-  val index: SurfaceIndex,
+  val chunkId: ServerMapChunkId,
-  val tiles: Map<MapTilePosition, PrototypeName>,
+  val map: Map<MapTilePosition, Data>,
-)
-
-/** Aggregate [KafkatorioPacket]s into a single [FactorioServerMap] (per [serverId][FactorioServerId]). */
-fun buildFactorioServerMap(
-  factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>
-): KTable<FactorioServerId, FactorioServerMap> {
-
-  return factorioServerPacketStream
-    .groupByKey(
-      groupedAs("factorio-server-map-data-grouping", jsonMapper.serde(), jsonMapper.serde())
-    )
-    .aggregate<FactorioServerId, KafkatorioPacket, FactorioServerMapAggregator>(
-      "factorio-server-map-data-aggregate",
-      materializedAs("factorio-server-map-aggregate-store", kxsBinary.serde(), kxsBinary.serde()),
-      { FactorioServerMapAggregator() },
-    ) { _: FactorioServerId, packet: KafkatorioPacket, aggregate: FactorioServerMapAggregator ->
-      aggregate + packet
-    }
-    .mapValues(
-      "finalise-factorio-server-map-aggregate",
-      materializedWith(kxsBinary.serde(), kxsBinary.serde())
-    ) { readOnlyKey, aggregate ->
-
-      val surfaces = aggregate.surfaces.mapValues { (_, surface) ->
-        FactorioMapSurface(surface.index, surface.tiles)
-      }
-
-      FactorioServerMap(readOnlyKey, aggregate.tilePrototypes, surfaces)
-    }
-
-}
-
-
-@Serializable
-private data class FactorioServerMapAggregator(
-  val tilePrototypes: MutableMap<PrototypeName, MapTilePrototype> = mutableMapOf(),
-  val surfaces: MutableMap<SurfaceIndex, FactorioMapSurfaceAggregator> = mutableMapOf(),
 ) {
+  operator fun plus(other: ServerMapChunkTiles<Data>) = copy(map = map + other.map)
+}
 
-  operator fun plus(packet: KafkatorioPacket): FactorioServerMapAggregator {
+
-    return when (packet) {
+fun groupTilesIntoChunksWithColours(
-      is FactorioEvent               -> {
+  factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>,
+  tileProtoColourDict: KTable<FactorioServerId, TileColourDict>,
+): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
+
+  // group tiles by server & surface & chunk
+  val chunkedTilesTable: KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>> =
+    factorioServerPacketStream
+      .flatMap("server-map-data.tiles.flatMapByChunk") { key: FactorioServerId, packet: KafkatorioPacket ->
+
+        val mapTileData = when (packet) {
+          is FactorioEvent ->
             when (val data = packet.data) {
-          is MapChunk -> plus(data)
+              is MapChunk -> data.tiles
-          is MapTiles -> plus(data)
+              is MapTiles -> data
-//        is MapTile  -> {} // hmmm... no surface index
+              else        -> null
-          else        -> this
-        }
-      }
-      is FactorioPrototypes          -> plus(packet)
-      is FactorioConfigurationUpdate -> this
             }
+          else             -> null
         }
 
-  operator fun plus(mapTiles: MapTiles): FactorioServerMapAggregator {
+        val zoomLevel = ZoomLevel.ZOOM_0
-//    println("FactorioServerMapAggregator: adding ${mapTiles.tiles.size} MapTiles from surface ${mapTiles.surfaceIndex}")
 
-    val surfaceIndex = SurfaceIndex(mapTiles.surfaceIndex)
+//
+//        ZoomLevel
+//          .values
+//          .mapNotNull { zoomLevel ->
 
-    val surface = surfaces.getOrPut(surfaceIndex) { FactorioMapSurfaceAggregator(surfaceIndex) }
+        mapTileData
-
+          ?.tiles
-    val surfaceTiles = mapTiles.tiles.associate { it.position to PrototypeName(it.prototypeName) }
+          ?.groupBy { tile ->
-    surface.tiles.putAll(surfaceTiles)
+            tile.position.toMapChunkPosition(zoomLevel.chunkSize)
-
-    return this
           }
+          ?.map { (chunkPos, tiles) ->
 
-  operator fun plus(mapChunk: MapChunk): FactorioServerMapAggregator {
+            val chunkId = ServerMapChunkId(
-//    println("FactorioServerMapAggregator: adding mapChunk ${mapChunk.position}")
+              key,
-    return plus(mapChunk.tiles)
+              chunkPos,
-  }
+              SurfaceIndex(mapTileData.surfaceIndex),
-
+              zoomLevel,
-  operator fun plus(packet: FactorioPrototypes): FactorioServerMapAggregator {
-//    println("FactorioServerMapAggregator: adding FactorioPrototypes")
-
-    val packetPrototypes = packet.prototypes
-      .filterIsInstance<MapTilePrototype>()
-      .associateBy { PrototypeName(it.name) }
-
-    tilePrototypes.putAll(packetPrototypes)
-
-    return this
-  }
-
-}
-
-
-@Serializable
-private data class FactorioMapSurfaceAggregator(
-  val index: SurfaceIndex,
-  val tiles: MutableMap<MapTilePosition, PrototypeName> = mutableMapOf(),
             )
+
+            val tilesMap = tiles.associate { it.position to TileProtoHashCode(it) }
+
+            val chunkTiles = ServerMapChunkTiles(chunkId, tilesMap)
+
+            chunkId to chunkTiles
+          } ?: listOf()
+//          }.flatten()
+      }
+      .groupByKey(
+        groupedAs("server-map-data.tiles.group", kxsBinary.serde(), kxsBinary.serde())
+      )
+      .reduce(
+        "server-map-data.tiles.reduce",
+        materializedAs("server-map-data.tiles.store", kxsBinary.serde(), kxsBinary.serde()),
+      ) { a, b -> a + b }
+
+
+  val colourisedChunkTable: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
+    chunkedTilesTable
+      .join(
+        other = tileProtoColourDict,
+        name = "server-map-data.join-tiles-with-prototypes",
+        materialized = materializedWith(kxsBinary.serde(), kxsBinary.serde()),
+        foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<TileProtoHashCode> -> chunkTiles.chunkId.serverId }
+      ) { tiles: ServerMapChunkTiles<TileProtoHashCode>, colourDict: TileColourDict ->
+
+        ServerMapChunkTiles(
+          tiles.chunkId,
+          tiles.map.mapValues { (_, code) ->
+            colourDict.map.getOrElse(code) {
+              println("missing prototype $code")
+              ColourHex.TRANSPARENT
+            }
+          }
+        )
+      }
+
+  return colourisedChunkTable
+
+}
+
+//@Serializable
+//private data class FactorioServerMapAggregator(
+//  val tilePrototypes: SortedMap<PrototypeName, Colour> = TreeMap(),
+//  val surfaces: MutableMap<SurfaceIndex, FactorioMapSurfaceAggregator> = mutableMapOf(),
+//) {
+//
+//  operator fun plus(packet: KafkatorioPacket): FactorioServerMapAggregator {
+//    return when (packet) {
+//      is FactorioEvent               -> {
+//        when (val data = packet.data) {
+//          is MapChunk -> plus(data)
+//          is MapTiles -> plus(data)
+//          else        -> this
+//        }
+//      }
+//      is FactorioPrototypes          -> plus(packet)
+//      is FactorioConfigurationUpdate -> this
+//    }
+//  }
+//
+//  operator fun plus(mapTiles: MapTiles): FactorioServerMapAggregator {
+////    println("FactorioServerMapAggregator: adding ${mapTiles.tiles.size} MapTiles from surface ${mapTiles.surfaceIndex}")
+//
+//    val surfaceIndex = SurfaceIndex(mapTiles.surfaceIndex)
+//
+//    val surface = surfaces.getOrPut(surfaceIndex) { FactorioMapSurfaceAggregator(surfaceIndex) }
+//
+//    val surfaceTiles = mapTiles.tiles.associate { it.position to PrototypeName(it.prototypeName) }
+//    surface.tiles.putAll(surfaceTiles)
+//
+//    return this
+//  }
+//
+//  operator fun plus(mapChunk: MapChunk): FactorioServerMapAggregator {
+////    println("FactorioServerMapAggregator: adding mapChunk ${mapChunk.position}")
+//    return plus(mapChunk.tiles)
+//  }
+//
+//  operator fun plus(packet: FactorioPrototypes): FactorioServerMapAggregator {
+////    println("FactorioServerMapAggregator: adding FactorioPrototypes")
+//
+//    val packetPrototypes = packet.prototypes
+//      .filterIsInstance<MapTilePrototype>()
+//      .associate { PrototypeName(it.name) to it.mapColour }
+//
+//    tilePrototypes.putAll(packetPrototypes)
+//
+//    return this
+//  }
+//
+//}
+
+//
+//@Serializable
+//private data class FactorioMapSurfaceAggregator(
+//  val index: SurfaceIndex,
+//  val tiles: MutableMap<MapTilePosition, PrototypeName> = mutableMapOf(),
+//)

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/flowMapTiles.kt

@@ -3,10 +3,9 @@ package dev.adamko.kafkatorio.processor.topology
 import com.sksamuel.scrimage.ImmutableImage
 import com.sksamuel.scrimage.ScaleMethod
 import com.sksamuel.scrimage.nio.PngWriter
-import dev.adamko.kafkatorio.events.schema.Colour
+import dev.adamko.kafkatorio.events.schema.ColourHex
 import dev.adamko.kafkatorio.events.schema.MapChunkPosition
 import dev.adamko.kafkatorio.events.schema.MapTilePosition
-import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
 import java.awt.image.BufferedImage
 import java.io.File
 import kotlin.math.abs
@@ -15,17 +14,17 @@ import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.delay
+import kotlinx.coroutines.channels.BufferOverflow
-import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.MutableSharedFlow
-import kotlinx.coroutines.flow.asFlow
+import kotlinx.coroutines.flow.collect
 import kotlinx.coroutines.flow.conflate
-import kotlinx.coroutines.flow.filterNotNull
+import kotlinx.coroutines.flow.debounce
-import kotlinx.coroutines.flow.flatMapMerge
+import kotlinx.coroutines.flow.distinctUntilChanged
-import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.distinctUntilChangedBy
-import kotlinx.coroutines.flow.mapLatest
+import kotlinx.coroutines.flow.map
-import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.supervisorScope
 import org.apache.kafka.streams.kstream.KTable
 
 
@@ -36,106 +35,84 @@ private val cScope = CoroutineScope(
 )
 
 /** A bridge between Kafka and Kotlin */
-private val serverMapDataFlow: MutableStateFlow<FactorioServerMap?> = MutableStateFlow(null)
+private val chunkedTiles: MutableSharedFlow<ServerMapChunkTiles<ColourHex>> =
+  MutableSharedFlow(
+    0,
+    1,
+    BufferOverflow.SUSPEND
+  )
 
 
 fun saveMapTiles(
-  serverMapTable: KTable<FactorioServerId, FactorioServerMap>
+  serverMapTable: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>
 ) {
 
   serverMapTable
     .toStream()
     .foreach { _, value ->
+      if (value != null) {
         runBlocking(Dispatchers.Default) {
-        serverMapDataFlow.emit(value)
+          println("emitting chunkedTiles ${value.chunkId} tiles: ${value.map.size}")
+          chunkedTiles.emit(value)
+        }
       }
     }
 
   cScope.launch {
-    println("launching serverMapDataFlow merging")
+    supervisorScope {
-
+      chunkedTiles
-
-    serverMapDataFlow
-      .filterNotNull()
-
-    serverMapDataFlow
-      .filterNotNull()
         .conflate()
-//      .transform { serverMapData ->
+        .distinctUntilChanged()
-      .flatMapMerge(10) { serverMapData ->
+        .debounce(5.seconds)
-        println("flatMapMerge serverMapData ${serverMapData.serverId}, surfaces ${serverMapData.surfaces.keys}")
+        .map {
-
+          println("createMapTileImage ${it.chunkId}")
-        val prototypes = serverMapData.tilePrototypes
+          createMapTileImage(it)
-
+        }
-        serverMapData
+        .collect()
-          .surfaces
-          .values
-//          .forEach { surfaceData ->
-          .asFlow()
-          .flatMapMerge(10) { surfaceData ->
-            println("flatMapMerge surfaceData ${surfaceData.index}")
-
-            // get all tile colours for the surface
-            val surfaceTileColours = surfaceData.tiles
-              .entries
-              .mapNotNull { (tilePosition, prototypeName) ->
-                prototypes[prototypeName]?.let { prototype ->
-                  tilePosition to prototype.mapColour
     }
   }
-            println("surfaceTileColours.size ${surfaceTileColours.size}")
 
-            // split a surface into chunks, per zoom-level
+//  cScope.launch {
-            ZoomLevel.values
+//    println("launching serverMapDataFlow merging")
-//              .forEach { zoom ->
+//
-              .asFlow()
+//    chunkedTiles
-              .flatMapMerge(10) { zoom ->
+//      .runningFold(
-                println("flatMapMerge zoom $zoom")
+//        mapOf<ServerMapChunkId, MutableSharedFlow<ServerMapChunkTiles<ColourHex>>>()
-
+//      ) { accumulator, value ->
-                surfaceTileColours
+//
-                  .groupBy { (tilePosition, _) ->
+//        println("folding chunkedTiles ${value.chunkId}, current flows: ${accumulator.keys}")
-                    tilePosition.toMapChunkPosition(zoom.chunkSize)
+//
-                  }
+//        val chunkFlow = accumulator.getOrElse(value.chunkId) {
-                  .entries
+//
-//                    .forEach { (chunkPosition, tileColours) ->
+//          println("creating new MutableSharedFlow for ${value.chunkId}")
-                  .asFlow()
+//
-                  .mapLatest { (chunkPosition, tileColours) ->
+//          val input = MutableSharedFlow<ServerMapChunkTiles<ColourHex>>(
-                    println("mapLatest chunkPosition $chunkPosition")
+//            replay = 0,
-
+//            extraBufferCapacity = 1,
-                    // mapLatest + delay = an alternative to debounce-per-key?
+//            onBufferOverflow = BufferOverflow.DROP_OLDEST,
-                    delay(30.seconds)
+//          )
-
+//
-                    println("emitting ChunkData $chunkPosition")
+//          supervisorScope {
-
+//            println("launching MutableSharedFlow for ${value.chunkId}")
-                    ChunkData(
+//            input
-                      surfaceData.index,
+//              .debounce(30.seconds)
-                      zoom,
+//              .mapLatest {
-                      chunkPosition,
+//                println("createMapTileImage ${it.chunkId}")
-                      tileColours.toMap()
+//                createMapTileImage(it)
-                    )
+//              }.launchIn(this)
-                  }
+//          }
-              }
+//
-          }
+//          input
-      }
+//        }
-      .onEach { chunkData ->
+//
-
+//        chunkFlow.emit(value)
-        println("starting to save chunkData ${chunkData.chunkPosition}...")
+//
-
+//        accumulator + (value.chunkId to chunkFlow)
-        val chunkImageFile = File(chunkData.filename.value)
+//      }
-        if (chunkImageFile.parentFile.mkdirs()) {
+//      .collect()
-          println("created new map tile parentFile directory ${chunkImageFile.absolutePath}")
+//
-        }
+//    println("end of cScope.launch")
-
+//  }
-        val image =
-          createMapTileImage(chunkData.chunkPosition, chunkData.tileColours, chunkData.zoomLevel)
-
-        println("saving map tile $chunkImageFile")
-        val savedTile = image.output(PngWriter.NoCompression, chunkImageFile)
-        println("savedTile: $savedTile")
-
-      }
-      .launchIn(this)
-  }
 }
 
 
@@ -145,35 +122,58 @@ private value class TilePngFilename(
 ) {
 
   constructor(
-    surfaceIndex: SurfaceIndex,
+    chunkId: ServerMapChunkId
-    zoom: Int,
+  ) : this(buildString {
-    chunkX: Int,
+    append("src/main/resources/kafkatorio-web-map/s")
-    chunkY: Int,
+    append(chunkId.surfaceIndex.index)
-  ) : this("src/main/resources/kafkatorio-web-map/s${surfaceIndex.index}/z$zoom/x${chunkX}/y${chunkY}.png")
+    append("/z")
-
+    append(chunkId.zoomLevel.level)
-}
+    append("/x")
-
+    append(chunkId.chunkPosition.x)
-
+    append("/y")
-private data class ChunkData(
+    append(chunkId.chunkPosition.y)
-  val surfaceIndex: SurfaceIndex,
+    append(".png")
-  val zoomLevel: ZoomLevel,
+  })
-  val chunkPosition: MapChunkPosition,
-  val tileColours: Map<MapTilePosition, Colour>,
-) {
-  val filename = TilePngFilename(surfaceIndex, zoomLevel.level, chunkPosition.x, chunkPosition.y)
 }
 
+private fun createMapTileImage(
+  chunkTiles: ServerMapChunkTiles<ColourHex>
+) {
+
+  println("emitting ChunkData ${chunkTiles.chunkId.chunkPosition}")
+
+  println("starting to save chunkData ${chunkTiles.chunkId.chunkPosition}...")
+
+  val filename = TilePngFilename(chunkTiles.chunkId)
+
+  val chunkImageFile = File(filename.value)
+  if (chunkImageFile.parentFile.mkdirs()) {
+    println("created new map tile parentFile directory ${chunkImageFile.absolutePath}")
+  }
+
+  val image =
+    createMapTileImage(
+      chunkTiles.chunkId.chunkPosition,
+      chunkTiles.map,
+      chunkTiles.chunkId.zoomLevel
+    )
+
+  println("saving map tile $chunkImageFile")
+  val savedTile = image.output(PngWriter.NoCompression, chunkImageFile)
+  println("savedTile: $savedTile")
+}
+
+
 private fun createMapTileImage(
   chunkPosition: MapChunkPosition,
-  chunkColours: Map<MapTilePosition, Colour>,
+  chunkColours: Map<MapTilePosition, ColourHex>,
   z: ZoomLevel
 ): ImmutableImage {
 
-
   val chunkImage = ImmutableImage.filled(
     z.chunkSize,
     z.chunkSize,
-    COLOUR_TRANSPARENT.awt(),
+    ColourHex.TRANSPARENT.toRgbColor().awt(),
     BufferedImage.TYPE_INT_ARGB
   )
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/model.kt

@@ -2,6 +2,7 @@ package dev.adamko.kafkatorio.processor.topology
 
 import com.sksamuel.scrimage.color.RGBColor
 import dev.adamko.kafkatorio.events.schema.Colour
+import dev.adamko.kafkatorio.events.schema.ColourHex
 import kotlin.math.pow
 import kotlin.math.roundToInt
 import kotlinx.serialization.Serializable
@@ -23,12 +24,12 @@ value class FactorioServerId(val id: String) {
 }
 
 
-fun Colour.toRgbColor(): RGBColor {
+fun ColourHex.toRgbColor(): RGBColor {
   return RGBColor(
-    red.roundToInt(),
+    red.toInt(),
-    green.roundToInt(),
+    green.toInt(),
-    blue.roundToInt(),
+    blue.toInt(),
-    alpha.roundToInt(),
+    alpha.toInt(),
   )
 }
 
@@ -36,15 +37,14 @@ fun Colour.toRgbColor(): RGBColor {
 enum class ZoomLevel(
   val level: Int
 ) {
-  ZOOM_0(0),
+  ZOOM_0(0), // 512
-  ZOOM_1(1),
+  ZOOM_1(1), // 256
-  ZOOM_2(2),
+  ZOOM_2(2), // 128
-  ZOOM_3(3),
+  ZOOM_3(3), // 64
-  ZOOM_4(4),
+  ZOOM_4(4), // 32
-  ZOOM_5(5),
   ;
 
-  val chunkSize: Int = 2f.pow(10 - level).roundToInt()
+  val chunkSize: Int = 2f.pow(9 - level).roundToInt()
 
   init {
     require(chunkSize > 0) { "chunkSize must be positive" }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt

@@ -1,19 +1,17 @@
 package dev.adamko.kafkatorio.processor.topology
 
-import com.sksamuel.scrimage.color.RGBColor
+import dev.adamko.kafkatorio.events.schema.ColourHex
-import dev.adamko.kafkatorio.events.schema.Colour
 import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
 import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
+import dev.adamko.kafkatorio.events.schema.MapTile
 import dev.adamko.kafkatorio.events.schema.MapTilePrototype
 import dev.adamko.kafkatorio.processor.serdes.jsonMapper
-import dev.adamko.kafkatorio.processor.serdes.kxsBinary
+import dev.adamko.kotka.extensions.materializedAs
-import dev.adamko.kotka.extensions.consumedAs
+import dev.adamko.kotka.extensions.streams.mapValues
-import dev.adamko.kotka.extensions.materializedWith
+import dev.adamko.kotka.extensions.streams.toTable
-import dev.adamko.kotka.extensions.streams.flatMap
 import dev.adamko.kotka.kxs.serde
 import kotlinx.serialization.Serializable
-import org.apache.kafka.common.serialization.Serdes
+import org.apache.kafka.streams.kstream.KStream
-import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.kstream.KTable
 
 
@@ -24,50 +22,45 @@ value class PrototypeName(val name: String) {
 }
 
 
-//fun prototypesTable(builder: StreamsBuilder): KTable<PrototypeName, MapTilePrototype> {
+@JvmInline
-//  return builder.stream<FactorioPacketKey, FactorioPrototypes>(
+@Serializable
-//    "kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
+value class TileProtoHashCode(val code: Int) {
-//    consumedAs(
+  constructor(prototype: MapTilePrototype) : this(prototype.name.hashCode())
-//      "consume-all-prototypes-packets",
+  constructor(tile: MapTile) : this(tile.prototypeName.hashCode())
-//      jsonMapper.serde(),
+}
-//      jsonMapper.serde()
-//    )
-//  )
-//    .flatMap("map-MapTilePrototype") { _, prototypes: FactorioPrototypes ->
-//      prototypes.prototypes
-//        .filterIsInstance<MapTilePrototype>()
-//        .map { PrototypeName(it.name) to it }
-//    }
-//    .toTable(
-//      materializedWith(
-//        kxsBinary.serde(),
-//        kxsBinary.serde()
-//      )
-//    )
-//}
 
 
-/** Get the [Colour] of each [MapTilePrototype] */
+@JvmInline
-fun tilePrototypeColourTable(builder: StreamsBuilder): KTable<PrototypeName, Colour> {
+@Serializable
-  return builder.stream<FactorioServerId, FactorioPrototypes>(
+value class TileColourDict(val map: Map<TileProtoHashCode, ColourHex>) {
-    "kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
+  operator fun plus(other: TileColourDict) = TileColourDict(this.map + other.map)
-    consumedAs(
+}
-      "consume-all-prototypes-packets",
+
+
+/** Aggregate [KafkatorioPacket]s into a single [FactorioServerMap] (per [serverId][FactorioServerId]). */
+fun tileProtoColourDictionary(
+  factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>
+): KTable<FactorioServerId, TileColourDict> {
+
+  // get all the prototypes
+  return factorioServerPacketStream
+    .mapValues("server-map-data.tile-prototypes.mapValues") { _, packet ->
+      val map = when (packet) {
+        is FactorioPrototypes -> packet
+          .prototypes
+          .filterIsInstance<MapTilePrototype>()
+          .associate { TileProtoHashCode(it) to it.mapColour.toHex() }
+        else                  -> mapOf()
+      }
+      TileColourDict(map)
+    }
+    .toTable(
+      "server-map-data.tile-prototypes",
+      materializedAs(
+        "server-map-data.tile-prototypes.store",
         jsonMapper.serde(),
         jsonMapper.serde()
       )
     )
-    .flatMap("map-MapTilePrototype") { _, prototypes: FactorioPrototypes ->
+
-      prototypes.prototypes
-        .filterIsInstance<MapTilePrototype>()
-        .map {
-          PrototypeName(it.name) to  it.mapColour
-        }
-    }
-    .toTable(
-      materializedWith(
-        kxsBinary.serde(),
-        kxsBinary.serde()
-      )
-    )
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/webMapTiles.kt

@@ -13,89 +13,89 @@ import kotlin.math.abs
 import kotlin.math.roundToInt
 import org.apache.kafka.streams.kstream.KTable
 
-
-val COLOUR_TRANSPARENT = RGBColor.fromAwt(Color(0f, 0f, 0f, 0f))
-
-fun saveTileImages(
-  webMapTiles: KTable<WebMapTileChunkPosition, WebMapTileChunkPixels>
-) {
-
-  webMapTiles
-    .toStream("get-webmap-tile-updates")
-    .foreach { position: WebMapTileChunkPosition, pixels: WebMapTileChunkPixels ->
-      runCatching {
-        saveMapTilesPng(position, pixels.pixels)
-      }.onFailure { e ->
-        println("error saving map tile png chunk: $position")
-        e.printStackTrace()
-        throw e
-      }
-    }
-}
-
-//val saveMapTilesContext = Dispatchers.IO.limitedParallelism(1)
-
-
-private fun saveMapTilesPng(
-  chunkPosition: WebMapTileChunkPosition,
-  pixels: Map<MapTilePosition, Colour>,
-) {
-  val chunkOriginX: Int = chunkPosition.x * chunkPosition.chunkSize // - 1
-  val chunkOriginY: Int = chunkPosition.y * chunkPosition.chunkSize // - 1
-  val surfaceIndex: Int = chunkPosition.surfaceIndex.index
-
-  val chunkImage =
-    ImmutableImage.filled(
-      chunkPosition.chunkSize,
-      chunkPosition.chunkSize,
-      COLOUR_TRANSPARENT.awt(),
-      BufferedImage.TYPE_INT_ARGB
-    )
-
-  pixels.forEach { (tilePosition, colour) ->
-
-    val rgbColour = colour.toRgbColor()
-
-    val pixelX = abs(abs(tilePosition.x) - abs(chunkOriginX))
-    val pixelY = abs(abs(tilePosition.y) - abs(chunkOriginY))
-
-    chunkImage.setColor(
-      pixelX,
-      pixelY,
-      rgbColour
-    )
-
-  }
-
-  val zoom = 1u
-  val file =
-    File(tileFilename("$surfaceIndex", "$zoom", "${chunkPosition.x}", "${chunkPosition.y}"))
-
-  if (file.parentFile.mkdirs()) {
-    println("created new map tile parentFile directory ${file.absolutePath}")
-  }
-
-  println("saving map tile $file")
-  val savedTile = chunkImage.output(PngWriter.NoCompression, file)
-  println("savedTile: $savedTile")
-}
-
-fun tileFilename(
-  surfaceIndex: String,
-  zoom: String,
-  chunkX: String,
-  chunkY: String,
-) = "src/main/resources/kafkatorio-web-map/s${surfaceIndex}/z$zoom/x${chunkX}/y${chunkY}.png"
-
-//fun subdivide(tile: ImmutableImage, size: Int) {
 //
-//  val halfSize = size / 2
+//val COLOUR_TRANSPARENT: RGBColor = RGBColor.fromAwt(Color(0f, 0f, 0f, 0f))
 //
-//  val bl = tile.resizeTo(halfSize, halfSize, Position.BottomLeft)
+//fun saveTileImages(
-//  val savedTile = bl.output(PngWriter.NoCompression, file)
+//  webMapTiles: KTable<WebMapTileChunkPosition, WebMapTileChunkPixels>
+//) {
 //
-//  tile.resizeTo(halfSize, halfSize, Position.BottomRight)
+//  webMapTiles
-//  tile.resizeTo(halfSize, halfSize, Position.TopLeft)
+//    .toStream("get-webmap-tile-updates")
-//  tile.resizeTo(halfSize, halfSize, Position.TopRight)
+//    .foreach { position: WebMapTileChunkPosition, pixels: WebMapTileChunkPixels ->
+//      runCatching {
+//        saveMapTilesPng(position, pixels.pixels)
+//      }.onFailure { e ->
+//        println("error saving map tile png chunk: $position")
+//        e.printStackTrace()
+//        throw e
+//      }
+//    }
+//}
+//
+////val saveMapTilesContext = Dispatchers.IO.limitedParallelism(1)
+//
+//
+//private fun saveMapTilesPng(
+//  chunkPosition: WebMapTileChunkPosition,
+//  pixels: Map<MapTilePosition, Colour>,
+//) {
+//  val chunkOriginX: Int = chunkPosition.x * chunkPosition.chunkSize // - 1
+//  val chunkOriginY: Int = chunkPosition.y * chunkPosition.chunkSize // - 1
+//  val surfaceIndex: Int = chunkPosition.surfaceIndex.index
+//
+//  val chunkImage =
+//    ImmutableImage.filled(
+//      chunkPosition.chunkSize,
+//      chunkPosition.chunkSize,
+//      COLOUR_TRANSPARENT.awt(),
+//      BufferedImage.TYPE_INT_ARGB
+//    )
+//
+//  pixels.forEach { (tilePosition, colour) ->
+//
+//    val rgbColour = colour.toRgbColor()
+//
+//    val pixelX = abs(abs(tilePosition.x) - abs(chunkOriginX))
+//    val pixelY = abs(abs(tilePosition.y) - abs(chunkOriginY))
+//
+//    chunkImage.setColor(
+//      pixelX,
+//      pixelY,
+//      rgbColour
+//    )
 //
 //  }
+//
+//  val zoom = 1u
+//  val file =
+//    File(tileFilename("$surfaceIndex", "$zoom", "${chunkPosition.x}", "${chunkPosition.y}"))
+//
+//  if (file.parentFile.mkdirs()) {
+//    println("created new map tile parentFile directory ${file.absolutePath}")
+//  }
+//
+//  println("saving map tile $file")
+//  val savedTile = chunkImage.output(PngWriter.NoCompression, file)
+//  println("savedTile: $savedTile")
+//}
+//
+//fun tileFilename(
+//  surfaceIndex: String,
+//  zoom: String,
+//  chunkX: String,
+//  chunkY: String,
+//) = "src/main/resources/kafkatorio-web-map/s${surfaceIndex}/z$zoom/x${chunkX}/y${chunkY}.png"
+//
+////fun subdivide(tile: ImmutableImage, size: Int) {
+////
+////  val halfSize = size / 2
+////
+////  val bl = tile.resizeTo(halfSize, halfSize, Position.BottomLeft)
+////  val savedTile = bl.output(PngWriter.NoCompression, file)
+////
+////  tile.resizeTo(halfSize, halfSize, Position.BottomRight)
+////  tile.resizeTo(halfSize, halfSize, Position.TopLeft)
+////  tile.resizeTo(halfSize, halfSize, Position.TopRight)
+////
+////}

modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/serdeSize.kt

@@ -0,0 +1,62 @@
+package dev.adamko.kafkatorio.processor.misc
+
+import dev.adamko.kafkatorio.events.schema.ColourHex
+import dev.adamko.kafkatorio.events.schema.MapChunkPosition
+import dev.adamko.kafkatorio.events.schema.MapTilePosition
+import dev.adamko.kafkatorio.processor.serdes.kxsBinary
+import dev.adamko.kafkatorio.processor.topology.FactorioServerId
+import dev.adamko.kafkatorio.processor.topology.ServerMapChunkId
+import dev.adamko.kafkatorio.processor.topology.ServerMapChunkTiles
+import dev.adamko.kafkatorio.processor.topology.SurfaceIndex
+import dev.adamko.kafkatorio.processor.topology.ZoomLevel
+import java.util.UUID
+import kotlinx.serialization.encodeToByteArray
+import org.junit.jupiter.api.Test
+
+class Blah {
+
+  @Test
+  fun asdad() {
+
+    val tiles = ServerMapChunkTiles<ColourHex>(
+      chunkId = ServerMapChunkId(
+        serverId = FactorioServerId(UUID.randomUUID().toString()),
+        chunkPosition = MapChunkPosition(Int.MAX_VALUE, Int.MAX_VALUE),
+        surfaceIndex = SurfaceIndex(Int.MAX_VALUE),
+        zoomLevel = ZoomLevel.ZOOM_0
+      ),
+      buildMap {
+        repeat(512) { x ->
+          repeat(512) { y ->
+            put(MapTilePosition(x, y), ColourHex(255u, 255u, 255u, 255u))
+          }
+        }
+      }
+    )
+
+    val encode = kxsBinary.encodeToByteArray(tiles)
+
+    encode.inputStream()
+    val size = encode.size * Byte.SIZE_BYTES
+
+    println(size)
+
+
+    val map2 = buildList {
+      repeat(512) { x ->
+        repeat(512) { y ->
+          val element = (x to y) to Int.MAX_VALUE
+          add(element)
+        }
+      }
+    }
+//      .toTypedArray()
+    val encode2 = kxsBinary.encodeToByteArray(map2)
+
+    encode2.inputStream()
+    val size2 = encode2.size * Byte.SIZE_BYTES
+
+    println(size2)
+
+  }
+}

modules/infra-factorio-server/src/.env

@@ -1,4 +1,4 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
 FACTORIO_VERSION=1.1.53
-KAFKATORIO_VERSION=0.2.7
+KAFKATORIO_VERSION=0.2.8

modules/infra-kafka-cluster/src/.env

@@ -1,3 +1,3 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
-KAFKATORIO_VERSION=0.2.7
+KAFKATORIO_VERSION=0.2.8

modules/infra-kafka-cluster/src/kafka-server.properties

@@ -120,3 +120,8 @@ log.retention.check.interval.ms=300000
 ### Compression ###
 #compression.type=snappy
 #linger.ms=5
+
+
+### record size ###
+message.max.bytes=10485760
+replica.fetch.max.bytes=10485760

modules/infra-kafka-cluster/src/kafka.Dockerfile

@@ -1,4 +1,4 @@
-ARG KAFKA_DL_URL=https://dlcdn.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz
+ARG KAFKA_DL_URL=https://dlcdn.apache.org/kafka/3.1.0/kafka_2.13-3.1.0.tgz
 
 ## builder ##
 FROM debian:11-slim as Kafka-Download

modules/web-map/build.gradle.kts

@@ -4,7 +4,7 @@ import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackConfig
 plugins {
   id("dev.adamko.kafkatorio.lang.kotlin-js")
   kotlin("plugin.serialization")
-  id("io.kvision") version "5.7.1"
+  id("io.kvision") version "5.8.1"
 }
 
 kotlin {
@@ -48,8 +48,8 @@ kotlin {
 
         implementation(projects.modules.eventsSchema)
 
-        implementation("org.jetbrains.kotlinx:kotlinx-nodejs:0.0.7")
+        implementation(libs.kotlinx.nodejs)
-        implementation("org.jetbrains.kotlinx:kotlinx-html:0.7.3")
+        implementation(libs.kotlinx.html)
 
         kvision("kvision")
         kvision("kvision-bootstrap")