map tiles are now saved!

modules/events-mod/src/main/typescript/main/events/converters.ts

@@ -105,19 +105,10 @@ export namespace Converters {
     }
   }
 
-  // export function convertMapTilePositionToMapChunkPosition(
-  //     tilePosition: TilePositionTable
-  // ): ChunkPositionTable {
-  //   return {
-  //     x: tilePosition.x / 32,
-  //     y: tilePosition.y / 32,
-  //   }
-  // }
-
-  export function convertBoundingBox(bb: BoundingBoxTable): MapBoundingBox {
+  export function convertBoundingBox(bb: BoundingBoxRead): MapBoundingBox {
     return {
-      topLeft: mapTilePosition(bb.left_top as PositionTable),
-      bottomRight: mapTilePosition(bb.right_bottom as PositionTable),
+      topLeft: mapTilePosition(bb.left_top),
+      bottomRight: mapTilePosition(bb.right_bottom),
     }
   }
 

modules/events-mod/src/main/typescript/main/events/handlers.ts

@@ -57,14 +57,11 @@ export function handleChunkUpdate(
     eventType: string,
     surfaceIndex: uint,
     position: ChunkPositionTable,
-    area: BoundingBoxTable,
+    area: BoundingBoxRead,
 ) {
 
   let surface = game.get_surface(surfaceIndex)
 
-  // let x = (position as ChunkPositionTable).x * CHUNK_SIZE
-  // let y = (position as ChunkPositionTable).y * CHUNK_SIZE
-
   let tiles: LuaTile[] = surface?.find_tiles_filtered({
         area: area,
         // position: {x: x, y: y},
@@ -81,7 +78,7 @@ export function handleChunkUpdate(
     objectName: "MapChunk",
     tiles: createMapTiles(surfaceIndex, convertedTiles),
     position: Converters.mapChunkPosition(position),
-    area: Converters.convertBoundingBox(area)
+    area: Converters.convertBoundingBox(area),
   }
 
   emitEvent(mapChunk, tick, eventType)

modules/events-mod/src/main/typescript/main/events/registerHandlers.ts

@@ -9,6 +9,7 @@ import {
 import on_player_built_tile = defines.events.on_player_built_tile;
 import on_robot_built_tile = defines.events.on_robot_built_tile;
 
+
 const mapEventIdToName = new LuaTable<defines.Events, keyof typeof defines.events>()
 for (const [k, v] of pairs(defines.events)) {
   mapEventIdToName.set(v, k)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/converters/colour.kt

@@ -0,0 +1,64 @@
+package dev.adamko.kafkatorio.events.schema.converters
+
+import dev.adamko.kafkatorio.events.schema.Colour
+import kotlin.math.roundToInt
+
+
+/** True if any value is greater than 1, so the values are hexadecimal. */
+fun Colour.isHexadecimal(): Boolean {
+  return red > 1f
+      || green > 1f
+      || blue > 1f
+      || alpha > 1f
+}
+
+/** True if all values are between `[0..1]`. */
+fun Colour.isPercentage(): Boolean = !isHexadecimal()
+
+fun Colour.toHexadecimal() = if (isPercentage()) {
+  Colour(
+    (red * 255).coerceIn(0f..255f),
+    (green * 255).coerceIn(0f..255f),
+    (blue * 255).coerceIn(0f..255f),
+    (alpha * 255).coerceIn(0f..255f),
+  )
+} else {
+  this
+}
+
+fun Colour.toPercentage() = if (isPercentage()) {
+  this
+} else {
+  Colour(
+    (red / 255).coerceIn(0f..1f),
+    (green / 255).coerceIn(0f..1f),
+    (blue / 255).coerceIn(0f..1f),
+    (alpha / 255).coerceIn(0f..1f),
+  )
+}
+
+fun Colour.toHexString(includeAlpha: Boolean = false): String {
+
+  val hexConverter: Float.() -> String = if (isHexadecimal()) {
+    Float::decimalToHex
+  } else {
+    Float::percentageToHex
+  }
+
+  val length = if (includeAlpha) 9 else 7
+  return buildString(length) {
+    append('#')
+    append(red.hexConverter())
+    append(green.hexConverter())
+    append(blue.hexConverter())
+    if (includeAlpha) {
+      append(alpha.hexConverter())
+    }
+  }
+}
+
+fun Float.decimalToHex(): String =
+  roundToInt().toString(16).padStart(2, ' ')
+
+fun Float.percentageToHex(): String =
+  times(255).roundToInt().toString(16).padStart(2, ' ')

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/converters/positions.kt

@@ -0,0 +1,58 @@
+package dev.adamko.kafkatorio.events.schema.converters
+
+import dev.adamko.kafkatorio.events.schema.MAP_CHUNK_SIZE
+import dev.adamko.kafkatorio.events.schema.MapBoundingBox
+import dev.adamko.kafkatorio.events.schema.MapChunkPosition
+import dev.adamko.kafkatorio.events.schema.MapEntityPosition
+import dev.adamko.kafkatorio.events.schema.MapTilePosition
+import kotlin.math.floor
+
+operator fun MapTilePosition.times(factor: Int) =
+  MapTilePosition(x * factor, y * factor)
+
+operator fun MapTilePosition.div(divisor: Int) =
+  MapTilePosition(x / divisor, y / divisor)
+
+operator fun MapTilePosition.plus(addend: Int) =
+  MapTilePosition(x + addend, y + addend)
+
+
+fun MapTilePosition.toMapChunkPosition() =
+  MapChunkPosition(
+    floor(x.toDouble() / MAP_CHUNK_SIZE.toDouble()).toInt(),
+    floor(y.toDouble() / MAP_CHUNK_SIZE.toDouble()).toInt(),
+  )
+
+
+fun MapEntityPosition.toMapTilePosition() =
+  MapChunkPosition(floor(x).toInt(), floor(y).toInt())
+
+fun MapEntityPosition.toMapChunkPosition() =
+  MapChunkPosition(
+    floor(x / MAP_CHUNK_SIZE.toDouble()).toInt(),
+    floor(y / MAP_CHUNK_SIZE.toDouble()).toInt(),
+  )
+
+
+operator fun MapChunkPosition.times(factor: Int) =
+  MapChunkPosition(x * factor, y * factor)
+
+operator fun MapChunkPosition.plus(addend: Int) =
+  MapChunkPosition(x + addend, y + addend)
+
+operator fun MapChunkPosition.minus(subtrahend: Int) =
+  MapChunkPosition(x - subtrahend, y - subtrahend)
+
+
+val MapChunkPosition.leftTopTile: MapTilePosition
+  get() = MapTilePosition(
+    x * MAP_CHUNK_SIZE,
+    y * MAP_CHUNK_SIZE,
+  )
+
+val MapChunkPosition.rightBottomTile: MapTilePosition
+  get() = leftTopTile + (MAP_CHUNK_SIZE - 1)
+
+
+val MapChunkPosition.boundingBox: MapBoundingBox
+  get() = MapBoundingBox(leftTopTile, rightBottomTile)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt

@@ -1,9 +1,9 @@
 package dev.adamko.kafkatorio.events.schema
 
-import kotlin.math.roundToInt
 import kotlinx.serialization.EncodeDefault
 import kotlinx.serialization.Serializable
 
+
 const val MAP_CHUNK_SIZE = 32
 
 /**
@@ -16,14 +16,8 @@ const val MAP_CHUNK_SIZE = 32
 data class MapEntityPosition(
   val x: Double,
   val y: Double,
-) {
+)
 
-  fun toMapChunkPosition() = MapChunkPosition(
-    (x / MAP_CHUNK_SIZE).toInt(),
-    (y / MAP_CHUNK_SIZE).toInt(),
-  )
-
-}
 
 /** Coordinates of a chunk in a [SurfaceData] where each integer `x`/`y` represents a different
  * chunk.
@@ -35,19 +29,8 @@ data class MapEntityPosition(
 data class MapChunkPosition(
   val x: Int,
   val y: Int,
-) {
+)
 
-  fun toMapEntityPosition() = MapEntityPosition(
-    x * MAP_CHUNK_SIZE.toDouble(),
-    y * MAP_CHUNK_SIZE.toDouble(),
-  )
-
-  fun toMapTilePosition() = MapChunkPosition(
-    x * MAP_CHUNK_SIZE,
-    y * MAP_CHUNK_SIZE,
-  )
-
-}
 
 /**
  * Coordinates of a tile in a chunk on a [SurfaceData] where each integer `x`/`y` represents a
@@ -59,15 +42,7 @@ data class MapChunkPosition(
 data class MapTilePosition(
   val x: Int,
   val y: Int,
-) {
-  constructor(x: Number, y: Number) : this(x.toInt(), y.toInt())
-
-  fun toMapChunkPosition() = MapChunkPosition(
-    x / MAP_CHUNK_SIZE,
-    y / MAP_CHUNK_SIZE,
-  )
-
-}
+)
 
 
 @Serializable
@@ -93,39 +68,4 @@ data class Colour(
   val blue: Float = 0f,
   @EncodeDefault
   val alpha: Float = 1f,
-) {
-
-  /** True if any value is greater than 1, so the values are hexadecimal. */
-  fun isHexadecimal(): Boolean {
-    return red > 1f
-        || green > 1f
-        || blue > 1f
-        || alpha > 1f
-  }
-
-  /** True if all values are between `[0..1]`. */
-  fun isPercentage(): Boolean = !isHexadecimal()
-
-  fun asHexadecimal() = if (isPercentage()) {
-    Colour(
-      (red * 255).coerceIn(0f..255f),
-      (green * 255).coerceIn(0f..255f),
-      (blue * 255).coerceIn(0f..255f),
-      (alpha * 255).coerceIn(0f..255f),
-    )
-  } else {
-    this
-  }
-
-  fun asPercentage() = if (isPercentage()) {
-    this
-  } else {
-    Colour(
-      (red / 255).coerceIn(0f..1f),
-      (green / 255).coerceIn(0f..1f),
-      (blue / 255).coerceIn(0f..1f),
-      (alpha / 255).coerceIn(0f..1f),
-    )
-  }
-
-}
+)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioLuaObjects.kt

@@ -64,7 +64,7 @@ object FactorioObjectDataSerializer : JsonContentPolymorphicSerializer<FactorioO
       FactorioObjectData.ObjectName.LuaSurface         -> SurfaceData.serializer()
       FactorioObjectData.ObjectName.MapChunk           -> MapChunk.serializer()
       FactorioObjectData.ObjectName.LuaTile            -> MapTile.serializer()
-      FactorioObjectData.ObjectName.LuaTiles            -> MapTiles.serializer()
+      FactorioObjectData.ObjectName.LuaTiles           -> MapTiles.serializer()
       FactorioObjectData.ObjectName.ConsoleChatMessage -> ConsoleChatMessage.serializer()
       null                                             ->
         throw Exception("Unknown FactorioObjectData $key: '$type' ")

modules/events-server/.gitignore

@@ -0,0 +1 @@
+**/kafkatorio-web-map/**

modules/events-server/build.gradle.kts

@@ -37,7 +37,7 @@ dependencies {
 
   implementation("com.sksamuel.scrimage:scrimage-core:4.0.24")
 
-  val kotkaVersion = "0.0.5"
+  val kotkaVersion = "0.0.8"
   implementation("com.github.adamko-dev.kotka-streams:kotka-streams:$kotkaVersion") {
     isChanging = true
   }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/admin/CreateTopics.kt

@@ -29,8 +29,8 @@ object CreateTopics {
 
     KafkatorioPacket.PacketType.values().forEach { packetType ->
 
-      val kafkatorioTopics = buildList {
-        add(KafkatorioTopology.sourceTopic)
+      val kafkatorioTopics = buildSet {
+//        add(KafkatorioTopology.sourceTopic)
         when (packetType) {
           KafkatorioPacket.PacketType.EVENT      ->
             FactorioObjectData.ObjectName.values().forEach { objectName ->
@@ -72,7 +72,7 @@ object CreateTopics {
     replicationFactor: Short = 1,
     topicNames: () -> Collection<String>,
   ): CreateTopicsResult =
-    createTopics(topicNames().map {
+    createTopics(topicNames().distinct().map {
       NewTopic(it, numPartitions, replicationFactor)
     })
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt

@@ -15,25 +15,29 @@ import dev.adamko.kafkatorio.events.schema.MapTile
 import dev.adamko.kafkatorio.events.schema.MapTilePosition
 import dev.adamko.kafkatorio.events.schema.MapTilePrototype
 import dev.adamko.kafkatorio.events.schema.MapTiles
+import dev.adamko.kafkatorio.events.schema.converters.leftTopTile
+import dev.adamko.kafkatorio.events.schema.converters.rightBottomTile
+import dev.adamko.kafkatorio.events.schema.converters.toHexadecimal
+import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
 import dev.adamko.kotka.extensions.consumedAs
 import dev.adamko.kotka.extensions.groupedAs
 import dev.adamko.kotka.extensions.materializedAs
 import dev.adamko.kotka.extensions.producedAs
 import dev.adamko.kotka.extensions.streams.filter
 import dev.adamko.kotka.extensions.streams.flatMap
+import dev.adamko.kotka.extensions.streams.map
 import dev.adamko.kotka.extensions.streams.mapValues
 import dev.adamko.kotka.extensions.streams.merge
 import dev.adamko.kotka.extensions.streams.to
 import dev.adamko.kotka.kxs.serde
 import dev.adamko.kotka.topicdata.TopicRecord
 import dev.adamko.kotka.topicdata.flatMapTopicRecords
-import dev.adamko.kotka.topicdata.mapTopicRecords
 import java.awt.Color
 import java.awt.image.BufferedImage
 import java.io.File
 import java.time.Duration
+import kotlin.math.abs
 import kotlin.math.roundToInt
-import kotlin.math.sign
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
@@ -76,7 +80,7 @@ class KafkatorioTopology(
 
     streams.setUncaughtExceptionHandler(StreamsExceptionHandler())
 
-    Runtime.getRuntime().addShutdownHook(Thread { streams.close(Duration.ofSeconds(1)) })
+    Runtime.getRuntime().addShutdownHook(Thread { streams.close(Duration.ofSeconds(10)) })
 
 //  streams.cleanUp()
 
@@ -116,11 +120,11 @@ class KafkatorioTopology(
       ) { _: String, value: KafkatorioPacket, _: RecordContext ->
 //        println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}")
         when (value) {
-          is FactorioEvent ->
+          is FactorioEvent               ->
             "kafkatorio.${value.packetType.name}.${value.data.objectName.name}"
           is FactorioConfigurationUpdate ->
             "kafkatorio.${value.packetType.name}.FactorioConfigurationUpdate"
-          is FactorioPrototypes ->
+          is FactorioPrototypes          ->
             "kafkatorio.${value.packetType.name}.all"
         }
       }
@@ -139,6 +143,10 @@ class KafkatorioTopology(
   }
 
   private fun tilePrototypesTable(): KTable<String, MapTilePrototype> {
+
+//    val tilePrototypesStreamName = "kafkatorio.stream.prototypes-tiles"
+//    val tilePrototypesStoreName = "kafkatorio.store.prototypes-tiles"
+
     return builder.stream(
       "kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
       Consumed.with(Serdes.String(), jsonMapper.serde<FactorioPrototypes>())
@@ -158,6 +166,24 @@ class KafkatorioTopology(
           jsonMapper.serde()
         )
       )
+
+//      .to(
+//        tilePrototypesStreamName,
+//      )
+
+//    return builder.globalTable(
+//      tilePrototypesStreamName,
+//      consumedAs(
+//        "send-tile-protos-to-gkt-store",
+//        jsonMapper.serde(),
+//        jsonMapper.serde()
+//      ),
+//      materializedAs(
+//        storeName = tilePrototypesStoreName,
+//        keySerde = jsonMapper.serde(),
+//        valueSerde = jsonMapper.serde(),
+//      )
+//    )
   }
 
   @Serializable
@@ -168,14 +194,23 @@ class KafkatorioTopology(
 
   @Serializable
   data class MapChunkData(
-    val position: MapChunkPosition,
-    val surfaceIndex: Int,
+    val chunkPosition: MapChunkDataPosition,
     val tiles: Set<MapTile>,
-  ) : TopicRecord<MapChunkDataPosition> {
-    override val topicKey = MapChunkDataPosition(position, surfaceIndex)
+  )
+//    : TopicRecord<MapChunkDataPosition>
+  {
 
-    constructor(key: MapChunkDataPosition, tiles: Set<MapTile>) :
-        this(key.position, key.surfaceIndex, tiles)
+    //    @kotlinx.serialization.Transient
+//    override val topicKey by ::chunkPosition
+//    @kotlinx.serialization.Transient
+//    val surfaceIndex: Int by chunkPosition::surfaceIndex
+
+//    @kotlinx.serialization.Transient
+//    val leftTopTile = chunkPosition.position.leftTopTile
+//    @kotlinx.serialization.Transient
+//    val rightBottom = chunkPosition.position.rightBottomTile
+//    @kotlinx.serialization.Transient
+//    val leftBottom = MapTilePosition(leftTopTile.x, rightBottom.x)
   }
 
   private fun mapChunksStream() {
@@ -193,6 +228,7 @@ class KafkatorioTopology(
         println("MapChunk update $key, tiles count: ${value.tiles.size}")
       }
 
+
     val chunkTilesUpdateStream: KStream<String, MapTiles> = builder.stream(
       "kafkatorio.${KafkatorioPacket.PacketType.EVENT}.${FactorioObjectData.ObjectName.MapChunk}",
       Consumed.with(Serdes.String(), jsonMapper.serde<FactorioEvent>())
@@ -238,12 +274,12 @@ class KafkatorioTopology(
 
     val chunksTable: KTable<MapChunkDataPosition, MapChunkData> =
       allTileUpdatesStream
-        .mapTopicRecords("map-tile-updates-to-MapChunkData") { _: TileUpdateRecordKey, value: TileUpdateRecord ->
+        .map("map-tile-updates-to-MapChunkData") { _: TileUpdateRecordKey, value: TileUpdateRecord ->
           val key = MapChunkDataPosition(
             value.tilePosition.toMapChunkPosition(),
             value.surfaceIndex,
           )
-          MapChunkData(key, setOf(value.tile))
+          key to MapChunkData(key, setOf(value.tile))
         }
         // group all tiles by chunk position
         .groupByKey(
@@ -259,19 +295,18 @@ class KafkatorioTopology(
         runCatching {
           saveMapTilesPng(key, value)
         }.onFailure { e ->
-          println("error saving map tile png ${value.position}")
+          println("error saving map tile png chunk:${value.chunkPosition}")
           e.printStackTrace()
           throw e
         }
       }
   }
 
+  @Suppress("KotlinConstantConditions")
   private fun saveMapTilesPng(key: MapChunkDataPosition, chunk: MapChunkData) {
-    val (xChunk, yChunk) = chunk.position.toMapTilePosition()
-    val xOffset = when (xChunk.sign >= 0) {
-      true -> {}
-      false -> {}
-    }
+
+    val chunkOriginX = chunk.chunkPosition.position.leftTopTile.x
+    val chunkOriginY = chunk.chunkPosition.position.rightBottomTile.y
 
     val chunkImage =
       ImmutableImage.filled(
@@ -283,25 +318,28 @@ class KafkatorioTopology(
 
     chunk.tiles.forEach { tile ->
 
+      val prototypeColour = tilePrototypesStore
+        .get(tile.prototypeName)
+        ?.value()
+        ?.mapColour
+        ?.toHexadecimal()
 
-      val prototypeColour = tilePrototypesStore.get(tile.prototypeName)
-        .value()
-        .mapColour
-        .asHexadecimal()
+      val rgbColour = when (prototypeColour) {
+        null -> RGBColor(11, 11, 11, 0)
+        else -> RGBColor(
+          prototypeColour.red.roundToInt(),
+          prototypeColour.green.roundToInt(),
+          prototypeColour.blue.roundToInt(),
+          prototypeColour.alpha.roundToInt(),
+        )
+      }
 
-      val rgbColour = RGBColor(
-        prototypeColour.red.roundToInt(),
-        prototypeColour.green.roundToInt(),
-        prototypeColour.blue.roundToInt(),
-        prototypeColour.alpha.roundToInt(),
-      )
-
-      val x = (tile.position.x - xChunk) * xChunk.sign
-      val y = (tile.position.y - yChunk) * yChunk.sign
+      val pixelX = abs(abs(tile.position.x) - abs(chunkOriginX))
+      val pixelY = abs(abs(tile.position.y) - abs(chunkOriginY))
 
       chunkImage.setColor(
-        tile.position.x,
-        tile.position.y,
+        pixelX,
+        pixelY,
         rgbColour
       )
 
@@ -309,10 +347,10 @@ class KafkatorioTopology(
 
     val zoom = 1u
     val file =
-      File("kafkatorio-web-map/${key.surfaceIndex}/$zoom/${key.position.x}/${key.position.y}.png")
+      File("kafkatorio-web-map/s${key.surfaceIndex}/z$zoom/x${key.position.x}/y${key.position.y}.png")
 
-    require(file.mkdirs()) {
-      "error creating map tile directory ${file.absolutePath}"
+    if (file.parentFile.mkdirs()) {
+      println("created new map tile parentFile directory ${file.absolutePath}")
     }
 
     println("saving map tile $file")

modules/events-server/src/main/resources/external-annotations/org/apache/kafka/streams/state/annotations.xml

@@ -0,0 +1,5 @@
+<root>
+  <item name='org.apache.kafka.streams.state.ReadOnlyKeyValueStore V get(K)'>
+    <annotation name='org.jetbrains.annotations.Nullable'/>
+  </item>
+</root>

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/converters.kt

@@ -6,31 +6,4 @@ import io.kvision.maps.externals.leaflet.geo.LatLng
 import io.kvision.maps.Maps
 import kotlin.math.roundToInt
 
-
-fun Colour.toHexString(includeAlpha: Boolean = false): String {
-
-  val hexConverter: Float.() -> String = if (isDecimal()) {
-    Float::decimalToHex
-  } else {
-    Float::percentageToHex
-  }
-
-  val length = if (includeAlpha) 9 else 7
-  return buildString(length) {
-    append('#')
-    append(red.hexConverter())
-    append(green.hexConverter())
-    append(blue.hexConverter())
-    if (includeAlpha) {
-      append(alpha.hexConverter())
-    }
-  }
-}
-
-fun Float.decimalToHex(): String =
-  roundToInt().toString(16).padStart(2, ' ')
-
-fun Float.percentageToHex(): String =
-  times(255).roundToInt().toString(16).padStart(2, ' ')
-
 fun PlayerData.latLng(): LatLng = Maps.L.latLng((position.y * -1) to position.x)

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/state.kt

@@ -2,6 +2,7 @@ package dev.adamko.kafkatorio.webmap
 
 import dev.adamko.kafkatorio.events.schema.FactorioEvent
 import dev.adamko.kafkatorio.events.schema.PlayerData
+import dev.adamko.kafkatorio.events.schema.converters.toHexString
 import io.kvision.maps.externals.leaflet.layer.overlay.Tooltip
 import io.kvision.maps.externals.leaflet.layer.vector.CircleMarker
 import io.kvision.maps.Maps