biggest refactor ever...

- use kxs-ts-gen for TypeScript schemas
- update schema files, so they're tidier (no need for typescript-generator workarounds)
- version bumps (kotlin, gradle, tstl, factorio, kotest)
- improve kxs binary format (use okio)
- move schema files package ('events' wasn't needed)
- tidy up topic creation, naming. base it off classes, make util functions

.gitignore

@@ -3,6 +3,7 @@
 .secret.*
 
 .idea
+!.idea/icon.svg
 
 node_modules
 

build.gradle.kts

@@ -8,7 +8,7 @@ plugins {
 }
 
 group = "dev.adamko.kafkatorio"
-version = "0.3.2"
+version = "0.4.0"
 
 val licenseFile: RegularFile by extra(layout.projectDirectory.file("LICENSE"))
 
@@ -35,7 +35,7 @@ idea {
 }
 
 tasks.wrapper {
-  gradleVersion = "7.4.1"
+  gradleVersion = "7.4.2"
   distributionType = Wrapper.DistributionType.ALL
 }
 
@@ -57,3 +57,12 @@ val runKafkatorio by tasks.registering {
     ":modules:infra-factorio-server:processRestart",
   )
 }
+
+
+val runEventsServer by tasks.registering {
+  group = project.name
+
+  dependsOn(
+    ":modules:events-server:run",
+  )
+}

buildSrc/build.gradle.kts

@@ -12,7 +12,7 @@ object Versions {
   const val jvmTarget = "11"
   const val kotlinTarget = "1.6"
 
-  const val kotlin = "1.6.10"
+  const val kotlin = "1.6.20"
 
   const val gradleNodePlugin = "3.2.1"
   const val gradleDockerComposePlugin = "0.15.1"
@@ -22,7 +22,7 @@ object Versions {
   const val semver = "1.1.2"
   const val http4k = "4.19.1.0"
 
-  const val kotest = "5.1.0"
+  const val kotest = "5.2.3"
 }
 
 

buildSrc/repositories.settings.gradle.kts

@@ -10,6 +10,7 @@ dependencyResolutionManagement {
 //  repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS)
 
   repositories {
+    myMavenLocal()
     mavenCentral()
     jitpack()
     gradlePluginPortal()
@@ -31,6 +32,7 @@ dependencyResolutionManagement {
 
   pluginManagement {
     repositories {
+      myMavenLocal()
       gradlePluginPortal()
       mavenCentral()
       jitpack()
@@ -39,6 +41,20 @@ dependencyResolutionManagement {
 
 }
 
+
 fun RepositoryHandler.jitpack() {
   maven("https://jitpack.io")
 }
+
+
+fun RepositoryHandler.myMavenLocal(enabled: Boolean = true) {
+  if (enabled) {
+    logger.lifecycle("Maven local is enabled")
+    mavenLocal {
+      content {
+        includeGroup("dev.adamko")
+        includeGroup("dev.adamko.kxtsgen")
+      }
+    }
+  }
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/common.kt

@@ -16,11 +16,11 @@ object Versions {
   const val kotlinLang = "1.6"
   const val jvmTarget = "11"
 
-  const val node = "14.19.0"
+  const val node = "14.19.1"
 
-  const val kotlinWrappers = "0.0.1-pre.300-kotlin-1.6.10"
+  const val kotlinWrappers = "0.0.1-pre.330-kotlin-1.6.20"
   const val junit = "5.8.2"
-  const val kotest = "5.1.0"
+  const val kotest = "5.2.3"
   const val mockk = "1.12.1"
 }
 

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts

@@ -47,7 +47,6 @@ tasks.withType<KotlinCompile>().configureEach {
     "-opt-in=kotlin.RequiresOptIn",
     "-opt-in=kotlin.ExperimentalStdlibApi",
     "-opt-in=kotlin.time.ExperimentalTime",
-//    "-Xextended-compiler-checks",
 //    "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
 //    "-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
   )

gradle.properties

@@ -1,5 +1,5 @@
 kotlin.js.compiler=ir
 kotlin.mpp.stability.nowarn=true
-kotlin.mpp.enableGranularSourceSetsMetadata=true
+#kotlin.mpp.enableGranularSourceSetsMetadata=true
-kotlin.native.enableDependencyPropagation=false
+#kotlin.native.enableDependencyPropagation=false
 org.gradle.dependency.verification.console=verbose

gradle/libs.versions.toml

@@ -6,7 +6,7 @@
 
 ## compilation ##
 jvm        = "11"
-kotlin     = "1.6.10"
+kotlin     = "1.6.20"
 
 ## libs ##
 camel-kafkaConnectors = "0.11.0"
@@ -17,13 +17,14 @@ kvision               = "5.8.2"
 log4j2                = "2.17.2"
 qosLogback            = "1.2.11"
 slf4j                 = "1.7.36"
+okio                  = "3.1.0"
 
 ## test ##
 junit      = "5.8.2"
-kotest     = "5.1.0"
+kotest     = "5.2.3"
 
 ## infra ##
-factorio               = "1.1.53"
+factorio               = "1.1.57"
 confluent-kafkaConnect = "7.0.1"
 
 #########
@@ -53,9 +54,9 @@ kotlinx-serialization-protobuf = { group = "org.jetbrains.kotlinx", name = "kotl
 
 kotlinx-serialization-yaml     = { group = "com.charleskorn.kaml",  name = "kaml",                       version = "0.37.0"    }
 
-kotlinx-coroutines         = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core",    version = "1.6.0" }
+kotlinx-coroutines         = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core",    version = "1.6.1" }
 
-kotlinx-html               = { group = "org.jetbrains.kotlinx", name = "kotlinx-html",               version = "0.7.3"     }
+kotlinx-html               = { group = "org.jetbrains.kotlinx", name = "kotlinx-html",               version = "0.7.5"     }
 kotlinx-nodejs             = { group = "org.jetbrains.kotlinx", name = "kotlinx-nodejs",             version = "0.0.7"     }
 
 
@@ -103,6 +104,12 @@ skrapeit = { group = "it.skrape", name = "skrapeit", version = "1.2.0" }
 
 scrimage-core = { group = "com.sksamuel.scrimage", name = "scrimage-core", version = "4.0.31" }
 
+
+## okio ##
+okio-bom  = { group = "com.squareup.okio", name = "okio-bom", version.ref = "okio" }
+okio-core = { group = "com.squareup.okio", name = "okio" }
+
+
 #############
 ### test  ###
 #############

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

modules/events-mod/src/main/typescript/generated/kafkatorio-schema/config-update.d.ts

@@ -1,20 +0,0 @@
-// Generated by TypeScriptGenerator - do not edit this file manually
-
-interface FactorioModInfo {
-  currentVersion: string | null;
-  modName: string;
-  previousVersion: string | null;
-}
-
-interface FactorioGameDataUpdate {
-  newVersion: string | null;
-  oldVersion: string | null;
-}
-
-interface FactorioConfigurationUpdate extends KafkatorioPacket {
-  allMods: FactorioModInfo[];
-  factorioData: FactorioGameDataUpdate;
-  modVersion: string;
-  packetType: PacketType;
-  tick: uint;
-}

modules/events-mod/src/main/typescript/generated/kafkatorio-schema/kafkatorio-schema.ts

@@ -0,0 +1,217 @@
+// Generated by TypeScriptGenerator - do not edit this file manually
+
+export interface KafkatorioPacket2 {
+  modVersion: string;
+  tick: Tick;
+  data: KafkatorioPacketData2;
+}
+
+export type Tick = UInt;
+
+export type KafkatorioPacketData2 =
+  | KafkatorioPacketData2.ConfigurationUpdate
+  | KafkatorioPacketData2.ConsoleChatUpdate
+  | KafkatorioPacketData2.ConsoleCommandUpdate
+  | KafkatorioPacketData2.EntityUpdate
+  | KafkatorioPacketData2.MapChunkUpdate
+  | KafkatorioPacketData2.PlayerUpdate
+  | KafkatorioPacketData2.PrototypesUpdate
+  | KafkatorioPacketData2.SurfaceUpdate;
+
+export namespace KafkatorioPacketData2 {
+  export enum Type {
+    ConfigurationUpdate = "dev.adamko.kafkatorio.schema2.ConfigurationUpdate",
+    ConsoleChatUpdate = "dev.adamko.kafkatorio.schema2.ConsoleChatUpdate",
+    ConsoleCommandUpdate = "dev.adamko.kafkatorio.schema2.ConsoleCommandUpdate",
+    PrototypesUpdate = "dev.adamko.kafkatorio.schema2.PrototypesUpdate",
+    SurfaceUpdate = "dev.adamko.kafkatorio.schema2.SurfaceUpdate",
+    EntityUpdate = "dev.adamko.kafkatorio.schema2.EntityUpdate",
+    MapChunkUpdate = "dev.adamko.kafkatorio.schema2.MapChunkUpdate",
+    PlayerUpdate = "dev.adamko.kafkatorio.schema2.PlayerUpdate",
+  }
+  
+  export interface ConfigurationUpdate {
+    type: KafkatorioPacketData2.Type.ConfigurationUpdate;
+    factorioData: ConfigurationUpdateGameData;
+    allMods: ConfigurationUpdateModData[];
+    modStartupSettingsChange: boolean;
+    migrationApplied: boolean;
+  }
+  
+  export interface ConsoleChatUpdate {
+    type: KafkatorioPacketData2.Type.ConsoleChatUpdate;
+    authorPlayerIndex: PlayerIndex | null;
+    content: string;
+  }
+  
+  export interface ConsoleCommandUpdate {
+    type: KafkatorioPacketData2.Type.ConsoleCommandUpdate;
+    authorPlayerIndex: PlayerIndex | null;
+    command: string;
+    parameters: string;
+  }
+  
+  export interface PrototypesUpdate {
+    type: KafkatorioPacketData2.Type.PrototypesUpdate;
+    prototypes: FactorioPrototype2[];
+  }
+  
+  export interface SurfaceUpdate {
+    type: KafkatorioPacketData2.Type.SurfaceUpdate;
+    index: SurfaceIndex;
+    daytime: Double;
+    name: string;
+  }
+  
+  export interface EntityUpdate {
+    type: KafkatorioPacketData2.Type.EntityUpdate;
+    key: EntityUpdateKey;
+    eventCounts?: { [key: string]: UInt } | null;
+    chunkPosition?: MapChunkPosition | null;
+    graphicsVariation?: UByte | null;
+    health?: Float | null;
+    isActive?: boolean | null;
+    isRotatable?: boolean | null;
+    lastUser?: UInt | null;
+    localisedDescription?: string | null;
+    localisedName?: string | null;
+    prototype?: PrototypeName | null;
+  }
+  
+  export interface MapChunkUpdate {
+    type: KafkatorioPacketData2.Type.MapChunkUpdate;
+    key: MapChunkUpdateKey;
+    eventCounts?: { [key: string]: UInt } | null;
+    player?: PlayerIndex | null;
+    robot?: EntityIdentifiersData | null;
+    force?: ForceIndex | null;
+    tileDictionary?: MapTileDictionary | null;
+    isDeleted?: boolean | null;
+  }
+  
+  export interface PlayerUpdate {
+    type: KafkatorioPacketData2.Type.PlayerUpdate;
+    key: PlayerUpdateKey;
+    eventCounts?: { [key: string]: UInt } | null;
+    characterUnitNumber?: UnitNumber | null;
+    chatColour?: Colour | null;
+    colour?: Colour | null;
+    name?: string | null;
+    afkTime?: Tick | null;
+    ticksToRespawn?: Tick | null;
+    forceIndex?: ForceIndex | null;
+    isAdmin?: boolean | null;
+    isConnected?: boolean | null;
+    isShowOnMap?: boolean | null;
+    isSpectator?: boolean | null;
+    lastOnline?: Tick | null;
+    onlineTime?: Tick | null;
+    position?: MapEntityPosition | null;
+    surfaceIndex?: SurfaceIndex | null;
+    tag?: string | null;
+    diedCause?: EntityIdentifiersData | null;
+    bannedReason?: string | null;
+    kickedReason?: string | null;
+    disconnectReason?: string | null;
+    isRemoved?: boolean | null;
+  }
+}
+
+export type UInt = uint;
+
+export interface ConfigurationUpdateGameData {
+  oldVersion?: string | null;
+  newVersion?: string | null;
+}
+
+export type PlayerIndex = UInt;
+
+export type SurfaceIndex = UInt;
+
+export type Double = double;
+
+export interface EntityUpdateKey {
+  unitNumber: UnitNumber;
+  name: string;
+  protoType: string;
+}
+
+export type MapChunkPosition = [
+  x: Int,
+  y: Int,
+];
+
+export type UByte = uint8;
+
+export type Float = float;
+
+export type PrototypeName = string;
+
+export interface MapChunkUpdateKey {
+  chunkPosition: MapChunkPosition;
+  surfaceIndex: SurfaceIndex;
+}
+
+export interface EntityIdentifiersData {
+  unitNumber?: UnitNumber | null;
+  name: string;
+  protoType: string;
+}
+
+export type ForceIndex = UInt;
+
+export interface MapTileDictionary {
+  tilesXY: { [key: string]: { [key: string]: PrototypeKey } };
+  protos: { [key: PrototypeName]: PrototypeKey };
+}
+
+export interface PlayerUpdateKey {
+  index: PlayerIndex;
+}
+
+export type UnitNumber = UInt;
+
+export interface Colour {
+  red?: Float;
+  green?: Float;
+  blue?: Float;
+  alpha?: Float;
+}
+
+export type MapEntityPosition = [
+  x: Double,
+  y: Double,
+];
+
+export type Int = int;
+
+export interface ConfigurationUpdateModData {
+  modName: string;
+  currentVersion?: string | null;
+  previousVersion?: string | null;
+}
+
+export type FactorioPrototype2 =
+  | FactorioPrototype2.MapTile;
+
+export namespace FactorioPrototype2 {
+  export enum Type {
+    MapTile = "dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2.MapTile",
+  }
+  
+  export interface MapTile {
+    type: FactorioPrototype2.Type.MapTile;
+    name: PrototypeName;
+    layer: UInt;
+    mapColour: Colour;
+    collisionMasks: List;
+    order: string;
+    canBeMined: boolean;
+  }
+}
+
+export type Byte = int8;
+
+export type List = any;
+
+export type PrototypeKey = Int;

modules/events-mod/src/main/typescript/generated/kafkatorio-schema/object-data.d.ts

@@ -1,20 +0,0 @@
-// Generated by TypeScriptGenerator - do not edit this file manually
-
-type ObjectName = "LuaSurface" | "ConsoleChatMessage";
-
-interface FactorioObjectData {
-  objectName: ObjectName;
-}
-
-interface SurfaceData extends FactorioObjectData {
-  daytime: double;
-  index: uint;
-  name: string;
-  objectName: ObjectName;
-}
-
-interface ConsoleChatMessage extends FactorioObjectData {
-  authorPlayerIndex: uint | null;
-  content: string;
-  objectName: ObjectName;
-}

modules/events-mod/src/main/typescript/generated/kafkatorio-schema/prototype.d.ts

@@ -1,24 +0,0 @@
-// Generated by TypeScriptGenerator - do not edit this file manually
-
-type PrototypeObjectName = "LuaTilePrototype";
-
-interface FactorioPrototype {
-  prototypeObjectName: PrototypeObjectName;
-}
-
-interface FactorioPrototypes extends KafkatorioPacket {
-  modVersion: string;
-  packetType: PacketType;
-  prototypes: FactorioPrototype[];
-  tick: uint;
-}
-
-interface MapTilePrototype extends FactorioPrototype {
-  canBeMined: boolean;
-  collisionMasks: string[];
-  layer: uint;
-  mapColour: Colour;
-  name: string;
-  order: string;
-  prototypeObjectName: PrototypeObjectName;
-}

modules/events-mod/src/main/typescript/generated/kafkatorio-schema/schema.d.ts

@@ -1,51 +0,0 @@
-// Generated by TypeScriptGenerator - do not edit this file manually
-
-type PacketType = "EVENT" | "CONFIG" | "PROTOTYPES" | "UPDATE";
-
-interface KafkatorioPacket {
-  modVersion: string;
-  packetType: PacketType;
-  tick: uint;
-}
-
-interface FactorioEvent extends KafkatorioPacket {
-  data: FactorioObjectData;
-  eventType: string;
-  modVersion: string;
-  packetType: PacketType;
-  tick: uint;
-}
-
-interface Colour {
-  alpha: float;
-  blue: float;
-  green: float;
-  red: float;
-}
-
-interface EntityIdentifiers {
-  name: string;
-  type: string;
-  unitNumber: uint | null;
-}
-
-interface EntityIdentifiersData extends EntityIdentifiers {
-  name: string;
-  type: string;
-  unitNumber: uint | null;
-}
-
-interface MapEntityPosition {
-  x: double;
-  y: double;
-}
-
-interface MapChunkPosition {
-  x: int;
-  y: int;
-}
-
-interface MapTileDictionary {
-  protos: { [key: string]: string };
-  tilesXY: { [key: string]: { [key: string]: string } };
-}

modules/events-mod/src/main/typescript/generated/kafkatorio-schema/update.d.ts

@@ -1,90 +0,0 @@
-// Generated by TypeScriptGenerator - do not edit this file manually
-
-type FactorioEventUpdateType = "PLAYER" | "MAP_CHUNK" | "ENTITY";
-
-interface FactorioEventUpdateKey {
-  updateType: FactorioEventUpdateType;
-}
-
-interface FactorioEventUpdate extends FactorioEventUpdateKey {
-  eventCounts: { [key: string]: uint } | null;
-  updateType: FactorioEventUpdateType;
-}
-
-interface FactorioEventUpdatePacket extends KafkatorioPacket {
-  modVersion: string;
-  packetType: PacketType;
-  tick: uint;
-  update: FactorioEventUpdate;
-}
-
-interface PlayerUpdateKey extends FactorioEventUpdateKey {
-  index: uint;
-}
-
-interface PlayerUpdate extends FactorioEventUpdate, PlayerUpdateKey {
-  afkTime: uint | null;
-  bannedReason: string | null;
-  characterUnitNumber: uint | null;
-  chatColour: Colour | null;
-  colour: Colour | null;
-  diedCause: EntityIdentifiersData | null;
-  disconnectReason: string | null;
-  eventCounts: { [key: string]: uint } | null;
-  forceIndex: uint | null;
-  index: uint;
-  isAdmin: boolean | null;
-  isConnected: boolean | null;
-  isRemoved: boolean | null;
-  isShowOnMap: boolean | null;
-  isSpectator: boolean | null;
-  kickedReason: string | null;
-  lastOnline: uint | null;
-  name: string | null;
-  onlineTime: uint | null;
-  position: MapEntityPosition | null;
-  surfaceIndex: uint | null;
-  tag: string | null;
-  ticksToRespawn: uint | null;
-  updateType: FactorioEventUpdateType;
-}
-
-interface EntityUpdateKey extends EntityIdentifiers, FactorioEventUpdateKey {
-  name: string;
-  type: string;
-  unitNumber: uint;
-}
-
-interface EntityUpdate extends FactorioEventUpdate, EntityUpdateKey {
-  chunkPosition: MapChunkPosition | null;
-  eventCounts: { [key: string]: uint } | null;
-  graphicsVariation: uint8 | null;
-  health: float | null;
-  isActive: boolean | null;
-  isRotatable: boolean | null;
-  lastUser: uint | null;
-  localisedDescription: string | null;
-  localisedName: string | null;
-  name: string;
-  prototype: string | null;
-  type: string;
-  unitNumber: uint;
-  updateType: FactorioEventUpdateType;
-}
-
-interface MapChunkUpdateKey extends FactorioEventUpdateKey {
-  chunkPosition: MapChunkPosition;
-  surfaceIndex: uint;
-}
-
-interface MapChunkUpdate extends FactorioEventUpdate, MapChunkUpdateKey {
-  chunkPosition: MapChunkPosition;
-  eventCounts: { [key: string]: uint } | null;
-  force: uint | null;
-  isDeleted: boolean | null;
-  player: uint | null;
-  robot: EntityIdentifiersData | null;
-  surfaceIndex: uint;
-  tileDictionary: MapTileDictionary | null;
-  updateType: FactorioEventUpdateType;
-}

modules/events-mod/src/main/typescript/main/PacketEmitter.ts

@@ -0,0 +1,41 @@
+import {
+  KafkatorioPacket2,
+  KafkatorioPacketData2
+} from "../generated/kafkatorio-schema/kafkatorio-schema";
+import {KafkatorioKeyedPacketData} from "./types";
+
+export class KafkatorioPacketEmitter {
+
+  emitInstantPacket<DATA extends KafkatorioPacketData2>(
+      data: DATA,
+  ) {
+    KafkatorioPacketEmitter.emitPacket<KafkatorioPacket2>({
+      data: data,
+      modVersion: global.MOD_VERSION,
+      tick: game.tick,
+    })
+  }
+
+  emitKeyedPacket<DATA extends KafkatorioKeyedPacketData>(
+      data: DATA,
+  ) {
+    KafkatorioPacketEmitter.emitPacket<KafkatorioPacket2>({
+      data: data,
+      modVersion: global.MOD_VERSION,
+      tick: game.tick,
+    })
+  }
+
+  /** Emit a serialised KafkatorioPacket */
+  private static emitPacket<T extends KafkatorioPacket2>(packet: T) {
+    let data = game.table_to_json(packet)
+    localised_print(`KafkatorioPacket: ${data}`)
+    // rcon.print(`KafkatorioPacket: ${data}`)
+  }
+
+}
+
+
+const PacketEmitter = new KafkatorioPacketEmitter()
+
+export default PacketEmitter

modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts

@@ -1,10 +1,12 @@
 import KafkatorioSettings from "../settings/KafkatorioSettings";
+import {KafkatorioKeyedPacketData} from "../types";
 
 export namespace EventUpdates {
 
 
   declare const global: {
-    cache: LuaTable<string, CacheEntry<FactorioEventUpdateType>>,
+    /** Map the hash of the key, as a string, a cache entry */
+    cache: LuaTable<string, CacheEntry<KafkatorioKeyedPacketData>>,
   }
 
 
@@ -19,7 +21,7 @@ export namespace EventUpdates {
 
       if (force == true || isAnythingUndefined) {
         log("Initialising global.cache")
-        global.cache = new LuaTable<string, CacheEntry<FactorioEventUpdateType>>()
+        global.cache = new LuaTable<string, CacheEntry<KafkatorioKeyedPacketData>>()
       }
       log(`Finished initialising EventDataCache`)
     }
@@ -29,12 +31,13 @@ export namespace EventUpdates {
      * Update the cache data without restarting the 'time-to-emit' countdown, so that the data won't
      * be emitted more than once per {@link CacheEntry.expirationDurationTicks}.
      */
-    public throttle<TYPE extends FactorioEventUpdateType>(
+    public throttle<PACKET extends KafkatorioKeyedPacketData>(
-        key: CacheKey<TYPE>,
+        key: PacketKey<PACKET>,
-        mutate: CacheDataMutator<TYPE>,
+        type: PacketType<PACKET>,
+        mutate: CacheDataMutator<PACKET>,
         expirationDurationTicks?: uint,
     ) {
-      this.update(key, mutate, false, expirationDurationTicks)
+      this.update(key, type, mutate, false, expirationDurationTicks)
     }
 
 
@@ -42,23 +45,26 @@ export namespace EventUpdates {
      * Update the cache data and reset the 'time-to-emit' countdown, so the data won't be emitted
      * until there's an inactivity gap of {@link CacheEntry.expirationDurationTicks}.
      */
-    public debounce<TYPE extends FactorioEventUpdateType>(
+    public debounce<PACKET extends KafkatorioKeyedPacketData>(
-        key: CacheKey<TYPE>,
+        key: PacketKey<PACKET>,
-        mutate: CacheDataMutator<TYPE>,
+        type: PacketType<PACKET>,
+        mutate: CacheDataMutator<PACKET>,
         expirationDurationTicks?: uint,
     ) {
-      this.update(key, mutate, true, expirationDurationTicks)
+      this.update(key, type, mutate, true, expirationDurationTicks)
     }
 
 
-    private update<TYPE extends FactorioEventUpdateType>(
+    private update<PACKET extends KafkatorioKeyedPacketData>(
-        key: CacheKey<TYPE>,
+        key: PacketKey<PACKET>,
-        mutate: CacheDataMutator<TYPE>,
+        type: PacketType<PACKET>,
+        mutate: CacheDataMutator<PACKET>,
         resetLastUpdated: boolean,
         expirationDurationTicks?: uint,
     ) {
-      const entry: CacheEntry<TYPE> = this.getCacheEntry(key) ?? this.createCacheEntry(key, mutate)
+      const entry: CacheEntry<PACKET> = this.getCacheEntry(key) ??
-      mutate(entry.data)
+                                        Manager.createCacheEntry(key, type)
+      mutate(entry.packet)
       if (resetLastUpdated) {
         entry.lastUpdatedTick = game.tick
       }
@@ -72,8 +78,8 @@ export namespace EventUpdates {
      * Usage example: If an urgent update comes in, then the expiration can be reduced so that
      * it's transmitted sooner.
      */
-    public setExpiration(
+    public setExpiration<PACKET extends KafkatorioKeyedPacketData>(
-        key: CacheKey<FactorioEventUpdateType>,
+        key: PacketKey<PACKET>,
         expirationDurationTicks: uint,
     ) {
       const entry = this.getCacheEntry(key)
@@ -83,15 +89,16 @@ export namespace EventUpdates {
     }
 
 
-    public extractExpired(): Array<FactorioEventUpdate> {
+    // use in/out v4.7.0 https://github.com/microsoft/TypeScript/pull/48240
+    public extractExpired(): Array<KafkatorioKeyedPacketData> {
       let countExpired = 0
       let countTotal = 0
-      const expiredData: Array<FactorioEventUpdate> = []
+      const expiredData: Array<KafkatorioKeyedPacketData> = []
       for (let [key, entry] of global.cache) {
         countTotal++
         if (this.isExpired(entry)) {
           countExpired++
-          expiredData.push(entry.data)
+          expiredData.push(entry.packet)
           global.cache.delete(key)
         }
       }
@@ -102,15 +109,15 @@ export namespace EventUpdates {
     }
 
 
-    private getCacheEntry<TYPE extends FactorioEventUpdateType>(
+    private getCacheEntry<PACKET extends KafkatorioKeyedPacketData>(
-        key: CacheKey<TYPE>
+        key: PacketKey<PACKET>
-    ): CacheEntry<TYPE> | undefined {
+    ): CacheEntry<PACKET> | undefined {
-      const hash = game.encode_string(game.table_to_json(key))!!
+      const hash = Manager.hashKey(key)
       if (!global.cache.has(hash)) {
         return undefined
       }
       const value: CacheEntry<any> = global.cache.get(hash)
-      if (this.isEntryInstanceOf(value, key.updateType)) {
+      if (this.isEntryInstanceOf(value, key)) {
         return value
       } else {
         // Type mismatch. This shouldn't happen...
@@ -119,40 +126,53 @@ export namespace EventUpdates {
     }
 
 
-    private createCacheEntry<TYPE extends FactorioEventUpdateType>(
+    private static createCacheEntry<PACKET extends KafkatorioKeyedPacketData>(
-        key: CacheKey<TYPE>,
+        key: PacketKey<PACKET>,
-        mutate: CacheDataMutator<TYPE>,
+        type: PacketType<PACKET>,
-    ): CacheEntry<TYPE> {
+    ): CacheEntry<PACKET> {
-      const data: CacheData<TYPE> = <CacheData<TYPE>>{...key}
+      const data: PACKET = <PACKET>{
-      mutate(data)
+        type: type,
-      let entry: CacheEntry<TYPE> = new CacheEntry<TYPE>(data)
+        key: key,
-      const hash = game.encode_string(game.table_to_json(key))!!
+      }
+      let entry: CacheEntry<PACKET> = new CacheEntry<PACKET>(data)
+      const hash = Manager.hashKey(key)
       global.cache.set(hash, entry)
       return entry
     }
 
 
-    isExpired<TYPE extends FactorioEventUpdateType>(entry: CacheEntry<TYPE>): boolean {
+    isExpired<TYPE extends KafkatorioKeyedPacketData>(entry: CacheEntry<TYPE>): boolean {
-      const expiryDuration = entry.expirationDurationTicks
+      const expiryDuration =
-                             ??
+          entry.expirationDurationTicks
-                             KafkatorioSettings.getEventCacheExpirationTicks(entry.data.updateType)
+          ?? KafkatorioSettings.getEventCacheExpirationTicks(entry.packet)
+
+      if (expiryDuration == undefined) {
+        return true
+      } else {
         return game.tick - entry.lastUpdatedTick > expiryDuration
       }
-
-
-    isEntryInstanceOf<TYPE extends FactorioEventUpdateType>(
-        entry: CacheEntry<any> | undefined,
-        updateType: TYPE,
-    ): entry is CacheEntry<TYPE> {
-      return this.isDataInstanceOf<TYPE>(entry?.data, updateType)
     }
 
 
-    isDataInstanceOf<TYPE extends FactorioEventUpdateType>(
+    isEntryInstanceOf<PACKET extends KafkatorioKeyedPacketData>(
-        data: CacheData<any> | undefined,
+        entry: CacheEntry<any> | undefined,
-        updateType: TYPE,
+        key: PacketKey<PACKET>,
-    ): data is CacheData<TYPE> {
+    ): entry is CacheEntry<PACKET> {
-      return data != undefined && data.updateType == updateType
+      return this.isDataInstanceOf(entry?.packet, key)
+    }
+
+
+    isDataInstanceOf<PACKET extends KafkatorioKeyedPacketData>(
+        packet: PACKET | undefined,
+        key: PacketKey<PACKET>,
+    ): packet is PACKET {
+      return packet != undefined && packet.key == key
+    }
+
+    private static hashKey<PACKET extends KafkatorioKeyedPacketData>(
+        key: PacketKey<PACKET>,
+    ): string {
+      return game.encode_string(game.table_to_json(key))!!
     }
 
 
@@ -166,54 +186,56 @@ export namespace EventUpdates {
   }
 
 
-  /** Map a {@link FactorioEventUpdateType} to a {@link FactorioEventUpdate} DTO */
+  // export type KeyedPacketType = Exclude<KafkatorioPacketDataType,
-  type ConvertToUpdate<TYPE extends FactorioEventUpdateType> =
+  //     "CONFIG" | "CONSOLE_CHAT" | "CONSOLE_COMMAND" | "PROTOTYPES" | "SURFACE">
-      TYPE extends "PLAYER" ? PlayerUpdate :
+
-      TYPE extends "MAP_CHUNK" ? MapChunkUpdate :
+  // /** Map a {@link KeyedPacketType} to a {@link KafkatorioPacketData} DTO */
-      TYPE extends "ENTITY" ? EntityUpdate :
+  // type ConvertToUpdate<TYPE extends KafkatorioKeyedPacketData> =
-      never
+  //     TYPE extends "PLAYER" ? PlayerUpdate :
+  //     TYPE extends "MAP_CHUNK" ? MapChunkUpdate :
+  //     TYPE extends "ENTITY" ? EntityUpdate :
+  //     never
 
 
-  /** Map a {@link FactorioEventUpdateType} to a {@link FactorioEventUpdateKey} */
+  // /** Map a {@link KeyedPacketType} to a {@link KafkatorioKeyedPacketKey} */
-  type ConvertToUpdateKey<TYPE extends FactorioEventUpdateType> =
+  // type ConvertToUpdateKey<TYPE extends KafkatorioKeyedPacketData> =
-      TYPE extends "PLAYER" ? PlayerUpdateKey :
+  //     TYPE extends "PLAYER" ? PlayerUpdateKey :
-      TYPE extends "MAP_CHUNK" ? MapChunkUpdateKey :
+  //     TYPE extends "MAP_CHUNK" ? MapChunkUpdateKey :
-      TYPE extends "ENTITY" ? EntityUpdateKey :
+  //     TYPE extends "ENTITY" ? EntityUpdateKey :
-      never
+  //     never
 
 
-  /** Make `updateType` specific */
+  // /** Make `updateType` specific */
-  export type CacheKey<TYPE extends FactorioEventUpdateType> =
+  export type PacketKey<PACKET extends KafkatorioKeyedPacketData> = PACKET["key"]
-      CacheTyped<TYPE>
+  export type PacketType<PACKET extends KafkatorioKeyedPacketData> = PACKET["type"]
-      & Omit<ConvertToUpdateKey<TYPE>, "updateType">
+  //
+  //
+  // /** Make `updateType` specific */
+  // export type CacheData<TYPE extends KafkatorioKeyedPacketData> =
+  //     CacheTyped<TYPE>
+  //     & Omit<ConvertToUpdate<TYPE>, "updateType">
+  //
+  //
+  // export type CacheTyped<TYPE extends KafkatorioKeyedPacketData> = {
+  //   readonly updateType: TYPE
+  // }
 
 
-  /** Make `updateType` specific */
+  export type CacheDataMutator<PACKET extends KafkatorioKeyedPacketData> = (packet: PACKET) => void
-  export type CacheData<TYPE extends FactorioEventUpdateType> =
-      CacheTyped<TYPE>
-      & Omit<ConvertToUpdate<TYPE>, "updateType">
 
 
-  export type CacheTyped<TYPE extends FactorioEventUpdateType> = {
+  class CacheEntry<PACKET extends KafkatorioKeyedPacketData> {
-    readonly updateType: TYPE
-  }
-
-
-  export type CacheDataMutator<TYPE extends FactorioEventUpdateType> =
-      (data: CacheData<TYPE>) => void
-
-
-  class CacheEntry<TYPE extends FactorioEventUpdateType> {
     lastUpdatedTick: uint = game.tick
     expirationDurationTicks?: uint
 
-    data: CacheData<TYPE>
+    packet: PACKET
 
     constructor(
-        data: CacheData<TYPE>,
+        packet: PACKET,
         expirationDurationTicks?: uint,
     ) {
-      this.data = data
+
+      this.packet = packet
       this.expirationDurationTicks = expirationDurationTicks
     }
   }
@@ -222,4 +244,5 @@ export namespace EventUpdates {
 }
 
 const EventUpdatesManager = new EventUpdates.Manager()
+
 export default EventUpdatesManager

modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts

@@ -1,23 +1,16 @@
-import {emitPacket} from "../emitKafkatorioPacket";
 import EventCacheService from "./EventDataCache";
+import packetEmitter from "../PacketEmitter";
+import {KafkatorioKeyedPacketData} from "../types";
 
 /** Every 0.1 seconds */
 script.on_nth_tick(6, (nthTick: NthTickEventData) => {
-  const cachedEvents: Array<FactorioEventUpdate> = EventCacheService.extractExpired()
+  const cachedEvents: Array<KafkatorioKeyedPacketData> = EventCacheService.extractExpired()
 
   if (cachedEvents.length > 0) {
     log(`nth tick ${nthTick.tick} has ${cachedEvents.length} events`)
 
     for (const event of cachedEvents) {
-
+      packetEmitter.emitKeyedPacket(event)
-      const packet: FactorioEventUpdatePacket = {
-        tick: nthTick.tick,
-        modVersion: global.MOD_VERSION,
-        packetType: "UPDATE",
-        update: event,
-      }
-
-      emitPacket(packet)
     }
   }
 })

modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts

@@ -1,25 +1,33 @@
-import {emitPacket} from "../emitKafkatorioPacket";
+import PacketEmitter from "../PacketEmitter";
+import {
+  ConfigurationUpdateGameData,
+  ConfigurationUpdateModData,
+  KafkatorioPacketData2
+} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import Type = KafkatorioPacketData2.Type;
+import ConfigurationUpdate = KafkatorioPacketData2.ConfigurationUpdate;
 
 export function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
 
-  emitPacket<FactorioConfigurationUpdate>({
+  const configUpdateData: ConfigurationUpdate = {
-    modVersion: global.MOD_VERSION,
+    type: Type.ConfigurationUpdate,
-    packetType: "CONFIG",
+    migrationApplied: changeData.migration_applied,
+    modStartupSettingsChange: changeData.mod_startup_settings_changed,
     allMods: allMods(changeData),
-    factorioData: factorioData(changeData),
+    factorioData: factorioData(changeData)
-    tick: game.tick,
+  }
-  })
 
+  PacketEmitter.emitInstantPacket(configUpdateData)
 }
 
-function factorioData(changeData: ConfigurationChangedData): FactorioGameDataUpdate {
+function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdateGameData {
   return {
     oldVersion: changeData.old_version ?? null,
     newVersion: changeData.new_version ?? null,
   }
 }
 
-function allMods(e: ConfigurationChangedData): FactorioModInfo[] {
+function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] {
 
   let namesOfChangedMods: string[] = Object.keys(e.mod_changes)
   let namesOfCurrentMods: string[] = Object.keys(script.active_mods)
@@ -33,7 +41,7 @@ function allMods(e: ConfigurationChangedData): FactorioModInfo[] {
     let currentVer = script.active_mods[modName] ?? e.mod_changes[modName].new_version ?? null
     let previousVer = e.mod_changes[modName]?.old_version ?? null
 
-    return <FactorioModInfo>{
+    return <ConfigurationUpdateModData>{
       modName: modName,
       currentVersion: currentVer,
       previousVersion: previousVer,

modules/events-mod/src/main/typescript/main/config-update/prototypes.ts

@@ -1,30 +1,37 @@
 import {Converters} from "../events/converters";
-import {emitPacket} from "../emitKafkatorioPacket";
+import PacketEmitter from "../PacketEmitter";
+import {
+  FactorioPrototype2,
+  KafkatorioPacketData2
+} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import Type = KafkatorioPacketData2.Type;
+import PrototypesUpdate = KafkatorioPacketData2.PrototypesUpdate;
 
 
 export function emitPrototypes() {
-  emitPacket<FactorioPrototypes>({
+
-    modVersion: global.MOD_VERSION,
+  const data: PrototypesUpdate = {
-    packetType: "PROTOTYPES",
     prototypes: prototypes(),
-    tick: game.tick,
+    type: Type.PrototypesUpdate,
-  })
+  }
+
+  PacketEmitter.emitInstantPacket(data)
 }
 
-function prototypes(): FactorioPrototype[] {
+function prototypes(): FactorioPrototype2[] {
-  let prototypes: FactorioPrototype[] = []
+  let prototypes: FactorioPrototype2[] = []
 
   prototypes.push(...getMapTilePrototypes())
 
   return prototypes
 }
 
-function getMapTilePrototypes(): MapTilePrototype[] {
+function getMapTilePrototypes(): FactorioPrototype2.MapTile[] {
-  let tiles: MapTilePrototype[] = []
+  let tiles: FactorioPrototype2.MapTile[] = []
   for (let [, tile] of game.tile_prototypes) {
     tiles.push(
-        <MapTilePrototype>{
+        <FactorioPrototype2.MapTile>{
-          prototypeObjectName: tile.object_name,
+          type: FactorioPrototype2.Type.MapTile,
 
           name: tile.name,
           order: tile.order,

modules/events-mod/src/main/typescript/main/emitKafkatorioPacket.ts

@@ -1,6 +0,0 @@
-/** Emit a serialised KafkatorioPacket */
-export function emitPacket<T extends KafkatorioPacket>(packet: T) {
-  let data = game.table_to_json(packet)
-  localised_print(`KafkatorioPacket: ${data}`)
-  // rcon.print(`KafkatorioPacket: ${data}`)
-}

modules/events-mod/src/main/typescript/main/events/converters.ts

@@ -1,4 +1,5 @@
 import {EventName} from "../types";
+import {Colour} from "../../generated/kafkatorio-schema/kafkatorio-schema";
 
 export namespace Converters {
 
@@ -62,15 +63,6 @@ export namespace Converters {
   //   return result
   // }
 
-  export function surfaceToTable(surface: LuaSurface): SurfaceData {
-    return {
-      objectName: surface.object_name,
-
-      name: surface.name,
-      index: surface.index,
-      daytime: surface.daytime,
-    }
-  }
 
   // export function mapEntityPosition(mapPos: MapPositionTable): MapEntityPosition {
   //   return {
@@ -102,14 +94,6 @@ export namespace Converters {
     }
   }
 
-  export function consoleChat(content: string, playerIndex?: uint): ConsoleChatMessage {
-    return {
-      objectName: "ConsoleChatMessage",
-
-      authorPlayerIndex: playerIndex ?? null,
-      content: content
-    }
-  }
 
   // export function convertTile(tile: LuaTile): MapTile {
   //   return {

modules/events-mod/src/main/typescript/main/events/emitEvent.ts

@@ -1,12 +0,0 @@
-import {emitPacket} from "../emitKafkatorioPacket";
-
-/** Emit a serialised event */
-export function emitEvent<T extends FactorioObjectData>(eventData: T, tick: uint, eventType: string) {
-  emitPacket({
-    data: eventData,
-    packetType: "EVENT",
-    eventType: eventType,
-    modVersion: global.MOD_VERSION,
-    tick: tick,
-  })
-}

modules/events-mod/src/main/typescript/main/events/handlers.ts

@@ -1,5 +1,6 @@
-import {Converters} from "./converters"
+import PacketEmitter from "../PacketEmitter";
-import {emitEvent} from "./emitEvent";
+import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import Type = KafkatorioPacketData2.Type;
 
 // export function handlePlayerUpdate(tick: uint, eventType: string, playerIndex: uint) {
 //   let player: LuaPlayer = game.players[playerIndex]
@@ -29,24 +30,23 @@ import {emitEvent} from "./emitEvent";
 //   emitEvent(table, tick, eventType)
 // }
 
+
 export function handleSurfaceUpdate(
-    tick: uint,
+    event: OnTickEvent,
     eventType: string,
     surface: LuaSurface,
 ) {
-  let table = Converters.surfaceToTable(surface)
+  const update: KafkatorioPacketData2.SurfaceUpdate = {
-  emitEvent(table, tick, eventType)
+    type: Type.SurfaceUpdate,
+
+    name: surface.name,
+    index: surface.index,
+    daytime: surface.daytime
+  }
+
+  PacketEmitter.emitInstantPacket(update)
 }
 
-export function handleConsoleChat(
-    tick: uint,
-    eventType: string,
-    playerIndex: uint | undefined,
-    message: string,
-) {
-  let table = Converters.consoleChat(message, playerIndex)
-  emitEvent(table, tick, eventType)
-}
 
 // export function handleChunkUpdate(
 //     tick: uint,

modules/events-mod/src/main/typescript/main/events/onConsoleEventHandlers.ts

@@ -0,0 +1,31 @@
+import PacketEmitter from "../PacketEmitter";
+import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import Type = KafkatorioPacketData2.Type;
+
+script.on_event(
+    defines.events.on_console_chat,
+    (event: OnConsoleChatEvent) => {
+      const update: KafkatorioPacketData2.ConsoleChatUpdate = {
+        type: Type.ConsoleChatUpdate,
+
+        authorPlayerIndex: event.player_index ?? null,
+        content: event.message
+      }
+      PacketEmitter.emitInstantPacket(update)
+    }
+)
+
+script.on_event(
+    defines.events.on_console_command,
+    (event: OnConsoleCommandEvent) => {
+      const update: KafkatorioPacketData2.ConsoleCommandUpdate = {
+        type: Type.ConsoleCommandUpdate,
+
+        authorPlayerIndex: event.player_index ?? null,
+        command: event.command,
+        parameters: event.parameters,
+      }
+
+      PacketEmitter.emitInstantPacket(update)
+    }
+)

modules/events-mod/src/main/typescript/main/events/onMapChunkEventListeners.ts

@@ -1,20 +1,25 @@
 import {isEventType} from "./eventTypeCheck";
 import {Converters} from "./converters";
 import EventUpdatesManager, {EventUpdates} from "../cache/EventDataCache";
-import CacheKey = EventUpdates.CacheKey;
+import {
-import CacheData = EventUpdates.CacheData;
+  KafkatorioPacketData2,
+  MapChunkPosition,
+  MapTileDictionary
+} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import CacheKey = EventUpdates.PacketKey;
+import Type = KafkatorioPacketData2.Type;
 
 
-type MapChunkUpdater = (data: CacheData<"MAP_CHUNK">) => void
+type MapChunkUpdater = (data: KafkatorioPacketData2.MapChunkUpdate) => void
 
 
-const mapProtoNameToKey: { [key: string]: string } = {}
+const mapProtoNameToKey: { [key: string]: uint } = {}
 let protoIndex: uint = 0
 
 
-function getProtoKey(protoName: string): string {
+function getProtoKey(protoName: string): uint {
   if (mapProtoNameToKey[protoName] == undefined) {
-    mapProtoNameToKey[protoName] = `${protoIndex++}`
+    mapProtoNameToKey[protoName] = protoIndex++
   }
   return mapProtoNameToKey[protoName]
 }
@@ -22,7 +27,7 @@ function getProtoKey(protoName: string): string {
 
 function mapTilesUpdateDebounce(
     surface: LuaSurface | undefined,
-    chunkPosition: ChunkPositionTable,
+    chunkPosition: MapChunkPosition,
     tiles: TileRead[],
     eventName: string,
     updater?: MapChunkUpdater,
@@ -33,14 +38,14 @@ function mapTilesUpdateDebounce(
     return
   }
 
-  const key: CacheKey<"MAP_CHUNK"> = {
+  const key: KafkatorioPacketData2.MapChunkUpdate["key"] = {
     surfaceIndex: surface.index,
     chunkPosition: chunkPosition,
-    updateType: "MAP_CHUNK",
   }
 
-  EventUpdatesManager.debounce<"MAP_CHUNK">(
+  EventUpdatesManager.debounce<KafkatorioPacketData2.MapChunkUpdate>(
       key,
+      Type.MapChunkUpdate,
       data => {
         data.tileDictionary ??= <MapTileDictionary>{
           tilesXY: {},
@@ -53,7 +58,7 @@ function mapTilesUpdateDebounce(
           const yString = `${tile.position.y}`
           const protoKey = getProtoKey(tile.name)
 
-          data.tileDictionary.protos[protoKey] = tile.name
+          data.tileDictionary.protos[tile.name] = protoIndex
 
           data.tileDictionary.tilesXY[xString] ??= {}
           data.tileDictionary.tilesXY[xString][yString] = protoKey
@@ -105,7 +110,7 @@ export function handleChunkGeneratedEvent(
   let tiles = getTiles(e.surface, e.area)
   mapTilesUpdateDebounce(
       e.surface,
-      e.position,
+      [e.position.x, e.position.y],
       tiles,
       Converters.eventNameString(e.name),
       undefined,
@@ -114,22 +119,22 @@ export function handleChunkGeneratedEvent(
 }
 
 
-script.on_event(
+// script.on_event(
-    defines.events.on_chunk_charted,
+//     defines.events.on_chunk_charted,
-    (e: OnChunkChartedEvent) => {
+//     (e: OnChunkChartedEvent) => {
-      log(`on_chunk_charted ${e.tick}`)
+//       log(`on_chunk_charted ${e.tick}`)
-
+//
-      // const surface = getSurface(e.surface_index)
+//       // const surface = getSurface(e.surface_index)
-      // if (surface == undefined) {
+//       // if (surface == undefined) {
-      //   return
+//       //   return
-      // }
+//       // }
-      //
+//       //
-      // let tiles = getTiles(surface, e.area)
+//       // let tiles = getTiles(surface, e.area)
-      // mapTilesUpdateDebounce(surface, e.position, tiles, (data => {
+//       // mapTilesUpdateDebounce(surface, e.position, tiles, (data => {
-      //   data.force = e.force.index
+//       //   data.force = e.force.index
-      // }))
+//       // }))
-    }
+//     }
-)
+// )
 
 
 script.on_event(
@@ -155,14 +160,19 @@ function groupTiles(tiles: TileRead[]): Map<MapChunkPosition, TileRead[]> {
 
   const mapChunkPositionToTiles = new Map<MapChunkPosition, TileRead[]>()
   for (const tile of tiles) {
-    const chunkPosition = {
+    const chunkPosition: MapChunkPosition = [
-      x: math.floor((tile.position.x / 32)),
+      math.floor((tile.position.x / 32)),
-      y: math.floor((tile.position.y / 32)),
+      math.floor((tile.position.y / 32)),
-    }
+    ]
     if (!mapChunkPositionToTiles.has(chunkPosition)) {
       mapChunkPositionToTiles.set(chunkPosition, [])
     }
-    mapChunkPositionToTiles.get(chunkPosition)?.push(tile)
+    // https://github.com/TypeScriptToLua/TypeScriptToLua/issues/1256
+    // mapChunkPositionToTiles.get(chunkPosition)?.push(tile)
+    const tiles = mapChunkPositionToTiles.get(chunkPosition)
+    if (tiles != null) {
+      tiles.push(tile)
+    }
   }
   return mapChunkPositionToTiles
 }
@@ -183,7 +193,7 @@ function convertOldPosition(
 }
 
 
-function onBuildTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent) {
+function handleBuiltTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent) {
 
   const surface = getSurface(event.surface_index)
   if (surface == undefined) {
@@ -209,7 +219,7 @@ function onBuildTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent)
                 data.robot = {
                   unitNumber: event.robot?.unit_number ?? null,
                   name: event.robot.name,
-                  type: event.robot.type,
+                  protoType: event.robot.type,
                 }
                 return
               }
@@ -225,28 +235,25 @@ function onBuildTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent)
 
 script.on_event(defines.events.on_player_built_tile, (e: OnPlayerBuiltTileEvent) => {
   log(`on_player_built_tile ${e.tick}`)
-  onBuildTileEvent(e)
+  handleBuiltTileEvent(e)
 })
 script.on_event(defines.events.on_robot_built_tile, (e: OnRobotBuiltTileEvent) => {
   log(`on_robot_built_tile ${e.tick}`)
-  onBuildTileEvent(e)
+  handleBuiltTileEvent(e)
 })
 
-// script.on_event(
-//     defines.events.on_player_mined_tile,
-//     (e: OnPlayerMinedTileEvent) => {
-//       // need to group tiles by ChunkPosition...
-//
-//     }
-// )
 
-// script.on_event(
+// function handleMinedTileEvent(event: OnPlayerMinedTileEvent | OnRobotMinedTileEvent) {
-//     defines.events.on_robot_mined_tile,
-//     (e: OnRobotMinedTileEvent) => {
-//       // need to group tiles by ChunkPosition...
-//
 // }
-// )
+//
+// script.on_event(defines.events.on_player_mined_tile, (e: OnPlayerMinedTileEvent) => {
+//   log(`on_player_mined_tile ${e.tick}`)
+//   handleMinedTileEvent(e)
+// })
+// script.on_event(defines.events.on_robot_mined_tile, (e: OnRobotMinedTileEvent) => {
+//   log(`on_robot_mined_tile ${e.tick}`)
+//   handleMinedTileEvent(e)
+// })
 
 
 script.on_event(
@@ -260,17 +267,15 @@ script.on_event(
       }
 
       for (const position of e.positions) {
-        const key: CacheKey<"MAP_CHUNK"> = {
+        const key: CacheKey<KafkatorioPacketData2.MapChunkUpdate> = {
           surfaceIndex: surface.index,
-          chunkPosition: position,
+          chunkPosition: [position.x, position.y],
-          updateType: "MAP_CHUNK",
         }
 
-        EventUpdatesManager.debounce<"MAP_CHUNK">(
+        EventUpdatesManager.debounce<KafkatorioPacketData2.MapChunkUpdate>(
             key,
-            data => {
+            Type.MapChunkUpdate,
-              data.isDeleted = true
+            data => data.isDeleted = true,
-            },
             0, // deletion is important - emit ASAP
         )
       }

modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts

@@ -1,17 +1,20 @@
 import {Converters} from "./converters";
-import EventUpdatesManager, {EventUpdates} from "../cache/EventDataCache";
+import EventUpdatesManager from "../cache/EventDataCache";
-import CacheData = EventUpdates.CacheData;
+import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import Type = KafkatorioPacketData2.Type;
 
 
-type PlayerUpdater = (player: LuaPlayer, data: CacheData<"PLAYER">) => void
+type PlayerUpdater = (player: LuaPlayer, data: KafkatorioPacketData2.PlayerUpdate) => void
 
 function playerUpdateThrottle(
     playerIndex: uint,
     eventName: string,
     mutate: PlayerUpdater,
 ) {
-  EventUpdatesManager.throttle<"PLAYER">(
+
-      {index: playerIndex, updateType: "PLAYER"},
+  EventUpdatesManager.throttle<KafkatorioPacketData2.PlayerUpdate>(
+      {index: playerIndex},
+      Type.PlayerUpdate,
       data => {
         const player = game.players[playerIndex]
         if (player != undefined) {
@@ -36,7 +39,7 @@ function playerUpdateThrottle(
 //   )
 // }
 
-function playerOnlineInfo(player: LuaPlayer, data: CacheData<"PLAYER">) {
+function playerOnlineInfo(player: LuaPlayer, data: KafkatorioPacketData2.PlayerUpdate) {
   data.lastOnline = player.last_online
   data.onlineTime = player.online_time
   data.afkTime = player.afk_time
@@ -73,7 +76,7 @@ script.on_event(
           e.player_index,
           Converters.eventNameString(e.name),
           (player, data) => {
-            data.position = player.position
+            data.position = [player.position.x, player.position.y]
           }
       )
     }
@@ -86,7 +89,7 @@ script.on_event(
           e.player_index,
           Converters.eventNameString(e.name),
           (player, data) => {
-            data.position = player.position
+            data.position = [player.position.x, player.position.y]
             data.surfaceIndex = player.surface.index
           }
       )
@@ -106,7 +109,7 @@ script.on_event(
               data.diedCause = {
                 unitNumber: e.cause.unit_number ?? null,
                 name: e.cause.name,
-                type: e.cause.type
+                protoType: e.cause.type
               }
             }
           }

modules/events-mod/src/main/typescript/main/events/registerHandlers.ts

@@ -1,9 +1,9 @@
-import {handleConsoleChat, handleSurfaceUpdate,} from "./handlers";
 import {Queue} from "../queue/queue";
 import {isEventType} from "./eventTypeCheck";
 import {Converters} from "./converters";
 import KafkatorioSettings from "../settings/KafkatorioSettings";
 import {handleChunkGeneratedEvent} from "./onMapChunkEventListeners";
+import {handleSurfaceUpdate} from "./handlers";
 
 
 // script.on_event(
@@ -34,13 +34,8 @@ script.on_event(
     (e: OnTickEvent) => {
       if (e.tick % 1000 == 0) {
         for (const [, surface] of game.surfaces) {
-          handleSurfaceUpdate(e.tick, Converters.eventNameString(e.name), surface)
+          handleSurfaceUpdate(e, Converters.eventNameString(e.name), surface)
         }
-
-        // let packets = new KafkatorioPacketQueue().dequeueValues(1)
-        // for (const packet of packets) {
-        //   emitPacket(packet)
-        // }
       }
 
       if (e.tick % 7 == 0) {
@@ -64,21 +59,6 @@ script.on_event(
     }
 )
 
-script.on_event(
-    defines.events.on_console_chat,
-    (e: OnConsoleChatEvent) => {
-      handleConsoleChat(e.tick, Converters.eventNameString(e.name), e.player_index, e.message)
-    }
-)
-
-// script.on_event(
-//     defines.events.on_chunk_generated,
-//     (e: OnChunkGeneratedEvent) => {
-//       handleChunkUpdate(e.tick, mapEventIdToName.get(e.name), e.surface.index, e.position,
-// e.area) } )  script.on_event( [ defines.events.on_player_built_tile,
-// defines.events.on_robot_built_tile, ], (builtTilesEvent) => { handleTilesUpdate(
-// builtTilesEvent.tick, mapEventIdToName.get(builtTilesEvent.name), builtTilesEvent.surface_index,
-// builtTilesEvent.tile, builtTilesEvent.tiles, ) } )
 
 script.on_event(
     defines.events.on_runtime_mod_setting_changed,

modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts

@@ -1,52 +1,57 @@
 import {Data, DoubleSettingDefinition} from "typed-factorio/settings/types"
+import {KafkatorioKeyedPacketData, KafkatorioKeyedPacketTypes} from "../types";
+import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
 
 export class KafkatorioSettingsConfig {
 
 
-  private static eventCacheExpirationDefaultSeconds: Record<FactorioEventUpdateType, uint> = {
+  private static eventCacheExpirationDefaultSeconds: Record<KafkatorioKeyedPacketTypes, double> = {
-    "PLAYER": 0.25,
+    "dev.adamko.kafkatorio.schema2.PlayerUpdate": 0.5,
-    "MAP_CHUNK": 30,
+    "dev.adamko.kafkatorio.schema2.MapChunkUpdate": 30,
-    "ENTITY": 5,
+    "dev.adamko.kafkatorio.schema2.EntityUpdate": 5,
   }
 
 
-  private eventCacheExpirationTicks: Record<FactorioEventUpdateType, double> = {
+  private eventCacheExpirationTicks: Record<KafkatorioKeyedPacketTypes, double> = {
-    "PLAYER": KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds["PLAYER"],
+    "dev.adamko.kafkatorio.schema2.PlayerUpdate":
-    "MAP_CHUNK": KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds["MAP_CHUNK"],
+        KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData2.Type.PlayerUpdate],
-    "ENTITY": KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds["ENTITY"],
+    "dev.adamko.kafkatorio.schema2.MapChunkUpdate":
+        KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData2.Type.MapChunkUpdate],
+    "dev.adamko.kafkatorio.schema2.EntityUpdate":
+        KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData2.Type.EntityUpdate],
   }
 
-
+  public getEventCacheExpirationTicks(packet: KafkatorioKeyedPacketData): uint {
-  public getEventCacheExpirationTicks(type: FactorioEventUpdateType): uint {
+    return this.eventCacheExpirationTicks[packet.type]
-    return this.eventCacheExpirationTicks[type]
   }
 
+  private static types: KafkatorioKeyedPacketTypes[] = [
+    KafkatorioPacketData2.Type.PlayerUpdate,
+    KafkatorioPacketData2.Type.MapChunkUpdate,
+    KafkatorioPacketData2.Type.EntityUpdate,
+  ]
 
-  loadSettings(): void {
+  public loadSettings(): void {
-    const types: FactorioEventUpdateType[] = ["PLAYER", "MAP_CHUNK", "ENTITY"]
+    for (const type of KafkatorioSettingsConfig.types) {
-    for (const type of types) {
       const seconds: double = settings.global[KafkatorioSettingsConfig.cacheDurationSettingName(type)].value as double
       this.eventCacheExpirationTicks[type] = seconds * 60 // convert ticks to seconds
     }
   }
 
 
-  initialiseSettings(data: Data): void {
+  public initialiseSettings(data: Data): void {
-    data.extend(
+    const settings = KafkatorioSettingsConfig.types.map(
-        [
+        type => KafkatorioSettingsConfig.createDefaultCacheDuration(type)
-          this.createDefaultCacheDuration("PLAYER"),
-          this.createDefaultCacheDuration("MAP_CHUNK"),
-          this.createDefaultCacheDuration("ENTITY"),
-        ]
     )
+    data.extend(settings)
   }
 
 
-  private createDefaultCacheDuration(
+  private static createDefaultCacheDuration(
-      type: FactorioEventUpdateType,
+      type: KafkatorioKeyedPacketTypes,
       defaultExpirationSeconds: uint = KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[type],
   ): DoubleSettingDefinition {
-    return <DoubleSettingDefinition>{
+    return {
       type: "double-setting",
       name: KafkatorioSettingsConfig.cacheDurationSettingName(type),
       setting_type: "runtime-global",
@@ -58,13 +63,12 @@ export class KafkatorioSettingsConfig {
   }
 
 
-  private static cacheDurationSettingName(type: FactorioEventUpdateType): string {
+  private static cacheDurationSettingName(type: KafkatorioKeyedPacketTypes): string {
-    return `kafkatorio_event-cache_default-expiration-duration-seconds_${type}`
+    return `kafkatorio_event-cache_default-expiration-duration-seconds_${type.replaceAll(".", "-")}`
   }
 
-
 }
 
-
 const KafkatorioSettings = new KafkatorioSettingsConfig()
+
 export default KafkatorioSettings

modules/events-mod/src/main/typescript/main/types.d.ts

@@ -1,3 +1,5 @@
+import {KafkatorioPacketData2} from "../generated/kafkatorio-schema/kafkatorio-schema";
+
 export type EventName = keyof typeof defines.events
 
 
@@ -15,3 +17,10 @@ type ExcludeNullKeys<TYPE> = Pick<TYPE, NonNullKeys<TYPE>>
 type NonNullableKeys<TYPE> = {
   [KEY in keyof TYPE]: NonNullable<TYPE[KEY]>;
 }
+
+
+type KafkatorioKeyedPacketData = Extract<KafkatorioPacketData2, { key: object }>
+
+type KafkatorioKeyedPacketTypes = KafkatorioKeyedPacketData["type"];
+
+type KafkatorioKeyedPacketKeys = KafkatorioKeyedPacketData["key"];

modules/events-mod/src/main/typescript/package-lock.json

@@ -1,13 +1,13 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
     "enhanced-resolve": {
-      "version": "5.9.2",
+      "version": "5.9.3",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.9.2.tgz",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.9.3.tgz",
-      "integrity": "sha512-GIm3fQfwLJ8YZx2smuHpBKkXC1yOk+OBEmKckVyL0i/ea8mqDEykK3ld5dgH1QYPNyT/lIllxV2LULnxCHaHkA==",
+      "integrity": "sha512-Bq9VSor+kjvW3f9/MiiR4eE3XYgOl7/rS8lnSxbRbF3kS0B2r+Y9w5krBWxZgDxASVZbdYrn5wT4j/Wb0J9qow==",
       "requires": {
         "graceful-fs": "^4.2.4",
         "tapable": "^2.2.0"
@@ -19,9 +19,9 @@
       "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "graceful-fs": {
-      "version": "4.2.9",
+      "version": "4.2.10",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
-      "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ=="
+      "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA=="
     },
     "has": {
       "version": "1.0.3",
@@ -75,19 +75,19 @@
       "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ=="
     },
     "typed-factorio": {
-      "version": "0.19.1",
+      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/typed-factorio/-/typed-factorio-0.19.1.tgz",
+      "resolved": "https://registry.npmjs.org/typed-factorio/-/typed-factorio-1.0.0.tgz",
-      "integrity": "sha512-/iLa916Cw33vC6wgmCb4cPLjmvUA2zVviTx/vy80iEvpOHjoNkuHkGSsmvss1aItwUKUlHaLi3k0fK5OPiw1iA=="
+      "integrity": "sha512-8JYdjizJKMUhEpaZ7SoVLQ+toe0XTebK/Zh/Iwwvs29x8Ml758XACX5Bn71LJK22u3obgu7YN2yn2GxZbQ554g=="
     },
     "typescript": {
-      "version": "4.6.2",
+      "version": "4.6.3",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.2.tgz",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.3.tgz",
-      "integrity": "sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg=="
+      "integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw=="
     },
     "typescript-to-lua": {
-      "version": "1.3.4",
+      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/typescript-to-lua/-/typescript-to-lua-1.3.4.tgz",
+      "resolved": "https://registry.npmjs.org/typescript-to-lua/-/typescript-to-lua-1.4.3.tgz",
-      "integrity": "sha512-9NkUOSUd9tufvwCGiWJ6Z1edizYcECUfjdZJZfDRKTEXKIddbAdV3IFR6u5gxvYYRfXhfNV6IKvqHS2HxHFs6w==",
+      "integrity": "sha512-oGJO1BqXKSxdfwavfOzthelxaQVAc7ykJ4Z/BF96nAWz35nmuLyv9FOSS132TgzLmV02859ndMuTYZtiVnN6jQ==",
       "requires": {
         "enhanced-resolve": "^5.8.2",
         "resolve": "^1.15.1",

modules/events-mod/src/main/typescript/package.json

@@ -1,28 +1,24 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "private": true,
   "scripts": {
     "build": "tstl",
     "dev": "tstl --watch"
   },
   "engines": {
-    "node": ">=10.13.0"
+    "node": "~14.19.1"
-  },
-  "devDependencies": {
   },
+  "devDependencies": {},
   "dependencies": {
     "lua-types": "^2.11.0",
-    "typescript-to-lua": "1.3.4",
+    "typescript-to-lua": "1.4.3",
-    "typed-factorio": "0.19.1",
+    "typed-factorio": "1.0.0",
-    "typescript": "4.6.2"
+    "typescript": "4.6.3"
   },
   "// @types/node": "^12.0.8",
   "license": "UNLICENSED",
-  "peerDependencies": {
+  "peerDependencies": {},
-  },
+  "optionalDependencies": {},
-  "optionalDependencies": {
+  "bundledDependencies": []
-  },
-  "bundledDependencies": [
-  ]
 }

modules/events-schema/build.gradle.kts

@@ -1,5 +1,6 @@
 import dev.adamko.kafkatorio.gradle.asProvider
 import dev.adamko.kafkatorio.gradle.typescriptAttributes
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 
 plugins {
@@ -11,6 +12,7 @@ plugins {
 
 
 kotlin {
+
   js(IR) {
     binaries.executable()
     // browser{} causes this error:
@@ -32,6 +34,11 @@ kotlin {
     nodejs()
   }
   jvm {
+    val main by compilations.getting {
+      kotlinOptions {
+        jvmTarget = "11"
+      }
+    }
     testRuns["test"].executionTask.configure {
       useJUnitPlatform()
     }
@@ -56,6 +63,10 @@ kotlin {
         implementation(project.dependencies.platform(libs.kotlinx.serialization.bom))
         implementation(libs.kotlinx.serialization.core)
         implementation(libs.kotlinx.serialization.json)
+        api("dev.adamko.kxtsgen:kxs-ts-gen-core:dev-SNAPSHOT")
+//        implementation("com.github.adamko-dev:kotlinx-serialization-typescript-generator:dev-SNAPSHOT") {
+//          isChanging = true
+//        }
       }
     }
 
@@ -76,7 +87,9 @@ kotlin {
       dependencies {
         implementation("com.github.aSemy:ts-generator:v1.2.1")
 //        implementation("com.github.ntrrgc:ts-generator:1.1.2")
-        implementation("org.jetbrains.kotlin:kotlin-reflect")
+        implementation(kotlin("reflect"))
+
+//        implementation("com.github.adamko-dev:kotlinx-serialization-typescript-generator:0.0.5")
       }
     }
 
@@ -105,7 +118,8 @@ val generateTypescript by tasks.registering(JavaExec::class) {
     main.runtimeDependencyFiles
   )
 
-  mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2tsKt")
+//  mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2tsKt")
+  mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2ts2Kt")
 
   val tempOutputDirectory = file("$temporaryDir")
   args(tempOutputDirectory)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/converters/positions.kt

@@ -1,57 +0,0 @@
-package dev.adamko.kafkatorio.events.schema.converters
-
-import dev.adamko.kafkatorio.events.schema.MAP_CHUNK_SIZE
-import dev.adamko.kafkatorio.events.schema.MapChunkPosition
-import dev.adamko.kafkatorio.events.schema.MapEntityPosition
-import dev.adamko.kafkatorio.events.schema.MapTilePosition
-import kotlin.math.floor
-
-operator fun MapTilePosition.times(factor: Int) =
-  MapTilePosition(x * factor, y * factor)
-
-operator fun MapTilePosition.div(divisor: Int) =
-  MapTilePosition(x / divisor, y / divisor)
-
-operator fun MapTilePosition.plus(addend: Int) =
-  MapTilePosition(x + addend, y + addend)
-
-
-fun MapTilePosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
-  MapChunkPosition(
-    floor(x.toDouble() / chunkSize.toDouble()).toInt(),
-    floor(y.toDouble() / chunkSize.toDouble()).toInt(),
-  )
-
-
-fun MapEntityPosition.toMapTilePosition() =
-  MapChunkPosition(floor(x).toInt(), floor(y).toInt())
-
-fun MapEntityPosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
-  MapChunkPosition(
-    floor(x / chunkSize.toDouble()).toInt(),
-    floor(y / chunkSize.toDouble()).toInt(),
-  )
-
-
-operator fun MapChunkPosition.times(factor: Int) =
-  MapChunkPosition(x * factor, y * factor)
-
-operator fun MapChunkPosition.plus(addend: Int) =
-  MapChunkPosition(x + addend, y + addend)
-
-operator fun MapChunkPosition.minus(subtrahend: Int) =
-  MapChunkPosition(x - subtrahend, y - subtrahend)
-
-
-fun MapChunkPosition.leftTopTile(chunkSize: Int = MAP_CHUNK_SIZE): MapTilePosition =
-  MapTilePosition(
-    x * chunkSize,
-    y * chunkSize,
-  )
-
-//val MapChunkPosition.rightBottomTile: MapTilePosition
-//  get() = leftTopTile + (MAP_CHUNK_SIZE - 1)
-//
-//
-//val MapChunkPosition.boundingBox: MapBoundingBox
-//  get() = MapBoundingBox(leftTopTile, rightBottomTile)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt

@@ -1,158 +0,0 @@
-package dev.adamko.kafkatorio.events.schema
-
-import kotlin.jvm.JvmInline
-import kotlinx.serialization.Serializable
-
-
-const val MAP_CHUNK_SIZE = 32
-
-
-@Serializable
-@JvmInline
-value class Tick(val value: UInt)
-
-
-/**
- * Coordinates of a [EntityData] on a map.
- *
- * A [MapEntityPosition] can be translated to a [MapChunkPosition] by dividing the `x`/`y` values
- * by [32][MAP_CHUNK_SIZE].
- */
-@Serializable
-data class MapEntityPosition(
-  val x: Double,
-  val y: Double,
-)
-
-
-/** Coordinates of a chunk in a [SurfaceData] where each integer `x`/`y` represents a different
- * chunk.
- *
- * A [MapChunkPosition] can be translated to a [MapEntityPosition] by multiplying the `x`/`y`
- * values by [32][MAP_CHUNK_SIZE].
- */
-@Serializable
-data class MapChunkPosition(
-  val x: Int,
-  val y: Int,
-)
-
-
-/**
- * Coordinates of a tile in a chunk on a [SurfaceData] where each integer `x`/`y` represents a
- * different [MapTile].
- *
- * It rounds any `x`/`y` down to whole numbers.
- */
-@Serializable
-data class MapTilePosition(
-  val x: Int,
-  val y: Int,
-)
-
-
-@Serializable
-data class MapBoundingBox(
-  val topLeft: MapTilePosition,
-  val bottomRight: MapTilePosition,
-)
-
-
-/**
- * Red, green, blue and alpha values, all in range `[0, 1]` or all in range `[0, 255]` if any
- * value is > 1.
- *
- * All values here are optional. Colour channels default to `0`, the alpha channel defaults to `1`.
- */
-@Serializable
-data class Colour(
-  val red: Float = 0f,
-  val green: Float = 0f,
-  val blue: Float = 0f,
-  val alpha: Float = 1f,
-)
-
-
-/** Size-efficient version of [Colour] (`4*4` bytes vs `4*1` bytes) */
-@Serializable
-data class ColourHex(
-  val red: UByte = UByte.MIN_VALUE,
-  val green: UByte = UByte.MIN_VALUE,
-  val blue: UByte = UByte.MIN_VALUE,
-  val alpha: UByte = UByte.MAX_VALUE,
-) {
-  companion object {
-    val TRANSPARENT = ColourHex(0u, 0u, 0u, 0u)
-  }
-}
-
-
-@Serializable
-data class MapTiles(
-  val surfaceIndex: SurfaceIndex,
-  val tiles: List<MapTile>,
-)
-
-
-@Serializable
-data class MapTile(
-  val x: Int,
-  val y: Int,
-  val proto: PrototypeName,
-)
-
-
-interface EntityIdentifiers {
-  val unitNumber: UnitNumber?
-  val name: String
-  val type: String
-}
-
-
-// instant of EntityIdentifiers, because kxs needs a concrete instance
-@Serializable
-data class EntityIdentifiersData(
-  override val unitNumber: UnitNumber? = null,
-  override val name: String,
-  /** The prototype-type of the entity. */
-  override val type: String,
-) : EntityIdentifiers
-
-
-/**
- * Size-optimised 2D map of x/y tile positions and prototype-name.
- *
- * Each X/Y coordinate maps to an arbitrary [PrototypeKey] (which is only valid for any specific
- * instance). [protos] is specific to each instance of [MapTileDictionary], can be used to convert from a
- */
-@Serializable
-data class MapTileDictionary(
-  /** Map an X,Y coordinate a prototype name */
-  val tilesXY: Map<String, Map<String, PrototypeKey>>,
-  val protos: Map<PrototypeKey, PrototypeName>,
-) {
-
-  fun toMapTileList(): List<MapTile> = buildList {
-    tilesXY.forEach { (xString, row) ->
-      row.forEach { (yString, protoIndex) ->
-        val x = xString.toIntOrNull()
-        val y = yString.toIntOrNull()
-        val protoName = protos[protoIndex]
-        if (x != null && y != null && protoName != null) {
-          add(MapTile(x, y, protoName))
-        }
-      }
-    }
-  }
-
-  companion object {
-    fun MapTileDictionary?.isNullOrEmpty(): Boolean {
-      return this == null || (tilesXY.isEmpty() && tilesXY.all { it.value.isEmpty() })
-    }
-  }
-}
-
-
-@Serializable
-@JvmInline
-value class PrototypeKey(val index: String)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioEventUpdates.kt

@@ -1,287 +0,0 @@
-package dev.adamko.kafkatorio.events.schema
-
-import dev.adamko.kafkatorio.events.schema.FactorioUpdateData.FactorioUpdateDataType
-import kotlinx.serialization.DeserializationStrategy
-import kotlinx.serialization.EncodeDefault
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.JsonContentPolymorphicSerializer
-import kotlinx.serialization.json.JsonElement
-import kotlinx.serialization.json.contentOrNull
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
-
-
-@Serializable
-sealed class KafkatorioPacket {
-  /** Schema versioning */
-  abstract val modVersion: String
-  abstract val tick: Tick
-  abstract val update: FactorioUpdateData
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-sealed interface FactorioUpdateDataKey {
-  val updateType: FactorioUpdateDataType
-}
-
-
-@Serializable(with = FactorioUpdateData.JsonSerializer::class)
-sealed class FactorioUpdateData : FactorioUpdateDataKey {
-
-  /** Discriminator for [FactorioUpdateData] */
-  @EncodeDefault
-  abstract override val updateType: FactorioUpdateDataType
-
-  /** the initial Factorio event (`defines.events`) trigger */
-  abstract val eventCounts: Map<String, UInt>?
-
-  @Serializable
-  enum class FactorioUpdateDataType {
-    PLAYER,
-    MAP_CHUNK,
-    ENTITY,
-    CONSOLE_CHAT,
-    CONSOLE_COMMAND,
-    SURFACE,
-    CONFIG,
-    PROTOTYPES,
-    ;
-
-    companion object {
-      val values: List<FactorioUpdateDataType> = values().toList()
-    }
-  }
-
-  object JsonSerializer : JsonContentPolymorphicSerializer<FactorioUpdateData>(
-    FactorioUpdateData::class
-  ) {
-    private val key = FactorioUpdateData::updateType.name
-
-    override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out FactorioUpdateData> {
-
-      val type = element
-        .jsonObject[key]
-        ?.jsonPrimitive
-        ?.contentOrNull
-        ?.let { json ->
-          FactorioUpdateDataType.values.firstOrNull { it.name == json }
-        }
-
-      requireNotNull(type) { "Unknown FactorioEventUpdate $key: $element" }
-
-      return when (type) {
-        FactorioUpdateDataType.PLAYER          -> PlayerUpdate.serializer()
-        FactorioUpdateDataType.MAP_CHUNK       -> MapChunkUpdate.serializer()
-        FactorioUpdateDataType.ENTITY          -> EntityUpdate.serializer()
-        FactorioUpdateDataType.CONSOLE_CHAT    -> ConsoleChatUpdate.serializer()
-        FactorioUpdateDataType.CONSOLE_COMMAND -> ConsoleCommandUpdate.serializer()
-        FactorioUpdateDataType.SURFACE         -> SurfaceUpdate.serializer()
-        FactorioUpdateDataType.CONFIG          -> ConfigurationUpdate.serializer()
-        FactorioUpdateDataType.PROTOTYPES      -> PrototypesUpdate.serializer()
-      }
-    }
-  }
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-sealed interface PlayerUpdateKey : FactorioUpdateDataKey {
-  val index: PlayerIndex
-}
-
-
-@Serializable
-data class PlayerUpdate(
-  override val index: PlayerIndex,
-
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val characterUnitNumber: UnitNumber? = null,
-  val chatColour: Colour? = null,
-  val colour: Colour? = null,
-  val name: String? = null,
-
-  val afkTime: Tick? = null,
-  val ticksToRespawn: Tick? = null,
-  val forceIndex: ForceIndex? = null,
-  val isAdmin: Boolean? = null,
-  val isConnected: Boolean? = null,
-  val isShowOnMap: Boolean? = null,
-  val isSpectator: Boolean? = null,
-  val lastOnline: Tick? = null,
-  val onlineTime: Tick? = null,
-  val position: MapEntityPosition? = null,
-  val surfaceIndex: SurfaceIndex? = null,
-  val tag: String? = null,
-  val diedCause: EntityIdentifiersData? = null,
-
-  val bannedReason: String? = null,
-  val kickedReason: String? = null,
-  val disconnectReason: String? = null,
-  /** `true` when a player is removed (deleted) from the game */
-  val isRemoved: Boolean? = null,
-) : FactorioUpdateData(), PlayerUpdateKey {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.PLAYER
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-sealed interface EntityUpdateKey : EntityIdentifiers, FactorioUpdateDataKey {
-  /** A [unitNumber] is required for caching */
-  override val unitNumber: UnitNumber
-  override val name: String
-  override val type: String
-}
-
-
-@Serializable
-data class EntityUpdate(
-  override val unitNumber: UnitNumber,
-  override val name: String,
-  override val type: String,
-
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val chunkPosition: MapChunkPosition? = null,
-  val graphicsVariation: UByte? = null,
-  val health: Float? = null,
-  val isActive: Boolean? = null,
-  val isRotatable: Boolean? = null,
-  val lastUser: UInt? = null,
-  val localisedDescription: String? = null,
-  val localisedName: String? = null,
-  val prototype: PrototypeName? = null,
-) : FactorioUpdateData(), EntityUpdateKey {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.ENTITY
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-sealed interface MapChunkUpdateKey : FactorioUpdateDataKey {
-  val chunkPosition: MapChunkPosition
-  val surfaceIndex: SurfaceIndex
-}
-
-
-@Serializable
-data class MapChunkUpdate(
-  override val chunkPosition: MapChunkPosition,
-  override val surfaceIndex: SurfaceIndex,
-
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val player: PlayerIndex? = null,
-  val robot: EntityIdentifiersData? = null,
-  val force: ForceIndex? = null,
-  /** updated tiles - might be partial and not all tiles in the chunk */
-  val tileDictionary: MapTileDictionary? = null,
-  val isDeleted: Boolean? = null,
-) : FactorioUpdateData(), MapChunkUpdateKey {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.MAP_CHUNK
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-@Serializable
-data class ConsoleChatUpdate(
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val authorPlayerIndex: PlayerIndex?,
-  val content: String,
-) : FactorioUpdateData() {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.CONSOLE_CHAT
-
-}
-
-@Serializable
-data class ConsoleCommandUpdate(
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val authorPlayerIndex: PlayerIndex?,
-  val command: String,
-  val parameters: String,
-) : FactorioUpdateData() {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.CONSOLE_COMMAND
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-sealed interface SurfaceUpdateKey : FactorioUpdateDataKey {
-  val index: SurfaceIndex
-}
-
-@Serializable
-data class SurfaceUpdate(
-  override val index: SurfaceIndex,
-
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val daytime: Double,
-  val name: String,
-) : FactorioUpdateData(), SurfaceUpdateKey {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.SURFACE
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-@Serializable
-data class ConfigurationUpdate(
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val factorioData: GameData,
-  val allMods: List<ModData>,
-) : FactorioUpdateData() {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.CONFIG
-
-  @Serializable
-  data class GameData(
-    val oldVersion: String? = null,
-    val newVersion: String? = null,
-  )
-
-  @Serializable
-  data class ModData(
-    val modName: String,
-    val currentVersion: String? = null,
-    val previousVersion: String? = null,
-  )
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
-
-
-@Serializable
-data class PrototypesUpdate(
-  override val eventCounts: Map<String, UInt>? = null,
-
-  val prototypes: List<FactorioPrototype>,
-) : FactorioUpdateData() {
-  @EncodeDefault
-  override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.PROTOTYPES
-}
-
-
-/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/colours.kt

@@ -1,8 +1,38 @@
-package dev.adamko.kafkatorio.events.schema.converters
+package dev.adamko.kafkatorio.schema.common
 
-import dev.adamko.kafkatorio.events.schema.Colour
-import dev.adamko.kafkatorio.events.schema.ColourHex
 import kotlin.math.roundToInt
+import kotlinx.serialization.Serializable
+
+/**
+ * Red, green, blue and alpha values, all in range `[0, 1]` or all in range `[0, 255]` if any
+ * value is > 1.
+ *
+ * All values here are optional. Colour channels default to `0`, the alpha channel defaults to `1`.
+ */
+@Serializable
+data class Colour(
+  val red: Float = 0f,
+  val green: Float = 0f,
+  val blue: Float = 0f,
+  val alpha: Float = 1f,
+)
+
+
+/** Size-efficient version of [Colour] (`4*4` bytes vs `4*1` bytes) */
+@Serializable
+data class ColourHex(
+  val red: UByte = UByte.MIN_VALUE,
+  val green: UByte = UByte.MIN_VALUE,
+  val blue: UByte = UByte.MIN_VALUE,
+  val alpha: UByte = UByte.MAX_VALUE,
+) {
+  companion object {
+    val TRANSPARENT = ColourHex(0u, 0u, 0u, 0u)
+  }
+}
+
+
+/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
 
 
 /** True if any value is greater than 1, so the values are hexadecimal. */
@@ -74,8 +104,10 @@ fun Colour.toHex(): ColourHex {
   )
 }
 
+
 private fun UByte.toPercentileFloat() = (this.toFloat() / ubyteMaxFloat).coerceIn(0f..1f)
 
+
 fun ColourHex.toPercentile(): Colour {
   return Colour(
     red.toPercentileFloat(),

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioIds.kt

@@ -1,9 +1,14 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema.common
 
 import kotlin.jvm.JvmInline
 import kotlinx.serialization.Serializable
 
 
+@Serializable
+@JvmInline
+value class Tick(val value: UInt)
+
+
 /**
  * Unique ID associated with this surface
  *
@@ -44,3 +49,20 @@ value class UnitNumber(private val id: UInt) {
 value class PrototypeName(private val id: String) {
   override fun toString(): String = id
 }
+
+
+interface EntityIdentifiers {
+  val unitNumber: UnitNumber?
+  val name: String
+  /** The prototype-type of the entity. */
+  val protoType: String
+}
+
+
+// instant of EntityIdentifiers, because kxs needs a concrete instance
+@Serializable
+data class EntityIdentifiersData(
+  override val unitNumber: UnitNumber? = null,
+  override val name: String,
+  override val protoType: String,
+) : EntityIdentifiers

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/mapTile.kt

@@ -0,0 +1,75 @@
+package dev.adamko.kafkatorio.schema.common
+
+import dev.adamko.kafkatorio.schema.common.MapTileDictionary.PrototypeKey
+import kotlin.jvm.JvmInline
+import kotlinx.serialization.Serializable
+
+
+const val MAP_CHUNK_SIZE = 32
+
+@Serializable
+data class MapTiles(
+  val surfaceIndex: SurfaceIndex,
+  val tiles: List<MapTile>,
+)
+
+
+@Serializable
+data class MapTile(
+  val x: Int,
+  val y: Int,
+  val proto: PrototypeName,
+)
+
+
+/**
+ * 2D map of x/y tile positions and prototype-name, optimised for JSON size.
+ *
+ * [protos] is a one-to-one map between a [PrototypeName] and [PrototypeKey].
+ *
+ * [PrototypeKey] is an arbitrary integer value, which maps to one [PrototypeName] only for a
+ * specific instance of [MapTileDictionary].
+ *
+ * Each X/Y coordinate in [tilesXY] maps to a [PrototypeKey].
+ */
+@Serializable
+data class MapTileDictionary(
+  /** Map an X,Y coordinate a prototype name */
+  val tilesXY: Map<String, Map<String, PrototypeKey>>,
+  val protos: Map<PrototypeName, PrototypeKey>,
+) {
+  private val protosIndexToName: Map<PrototypeKey, PrototypeName> by lazy {
+    protos.entries.associate { (name, key) -> key to name }
+  }
+
+  fun toMapTileList(): List<MapTile> = buildList {
+    tilesXY.forEach { (xString, row) ->
+      row.forEach { (yString, protoIndex) ->
+        val x = xString.toIntOrNull()
+        val y = yString.toIntOrNull()
+        val protoName = protosIndexToName[protoIndex]
+        if (x != null && y != null && protoName != null) {
+          add(MapTile(x, y, protoName))
+        }
+      }
+    }
+  }
+
+  /** The collection-index of a prototype in [MapTileDictionary.protos] */
+  @Serializable
+  @JvmInline
+  value class PrototypeKey(val index: Int)
+
+  companion object {
+    fun MapTileDictionary?.isNullOrEmpty(): Boolean {
+      return this == null || (tilesXY.isEmpty() && tilesXY.all { it.value.isEmpty() })
+    }
+  }
+}
+
+
+@Serializable
+data class MapBoundingBox(
+  val topLeft: MapTilePosition,
+  val bottomRight: MapTilePosition,
+)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/positions.kt

@@ -0,0 +1,142 @@
+package dev.adamko.kafkatorio.schema.common
+
+import dev.adamko.kxstsgen.core.*
+import dev.adamko.kxstsgen.core.experiments.TupleSerializer
+import kotlin.math.floor
+import kotlinx.serialization.Serializable
+
+/**
+ * Coordinates of a [EntityData] on a map.
+ *
+ * A [MapEntityPosition] can be translated to a [MapChunkPosition] by dividing the `x`/`y` values
+ * by [32][MAP_CHUNK_SIZE].
+ */
+@Serializable(with = MapEntityPosition.Serializer::class)
+data class MapEntityPosition(
+  val x: Double,
+  val y: Double,
+) {
+  object Serializer : TupleSerializer<MapEntityPosition>(
+    "MapEntityPosition",
+    {
+      element(MapEntityPosition::x)
+      element(MapEntityPosition::y)
+    }
+  ) {
+    override fun tupleConstructor(elements: Iterator<*>): MapEntityPosition {
+      return MapEntityPosition(
+        elements.next() as Double,
+        elements.next() as Double,
+      )
+    }
+  }
+}
+
+
+/** Coordinates of a chunk in a [SurfaceData] where each integer `x`/`y` represents a different
+ * chunk.
+ *
+ * A [MapChunkPosition] can be translated to a [MapEntityPosition] by multiplying the `x`/`y`
+ * values by [32][MAP_CHUNK_SIZE].
+ */
+@Serializable(with = MapChunkPosition.Serializer::class)
+data class MapChunkPosition(
+  val x: Int,
+  val y: Int,
+) {
+  object Serializer : TupleSerializer<MapChunkPosition>(
+    "MapChunkPosition",
+    {
+      element(MapChunkPosition::x)
+      element(MapChunkPosition::y)
+    }
+  ) {
+    override fun tupleConstructor(elements: Iterator<*>): MapChunkPosition {
+      return MapChunkPosition(
+        elements.next() as Int,
+        elements.next() as Int,
+      )
+    }
+  }
+}
+
+
+/**
+ * Coordinates of a tile in a chunk on a [SurfaceData] where each integer `x`/`y` represents a
+ * different [MapTile].
+ *
+ * It rounds any `x`/`y` down to whole numbers.
+ */
+@Serializable(with = MapTilePosition.Serializer::class)
+data class MapTilePosition(
+  val x: Int,
+  val y: Int,
+) {
+  object Serializer : TupleSerializer<MapTilePosition>(
+    "MapTilePosition",
+    {
+      element(MapTilePosition::x)
+      element(MapTilePosition::y)
+    }
+  ) {
+    override fun tupleConstructor(elements: Iterator<*>): MapTilePosition {
+      val x = requireNotNull(elements.next() as? Int)
+      val y = requireNotNull(elements.next() as? Int)
+      return MapTilePosition(x, y)
+    }
+  }
+}
+
+
+/* *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  *  * */
+
+
+operator fun MapTilePosition.times(factor: Int) =
+  MapTilePosition(x * factor, y * factor)
+
+operator fun MapTilePosition.div(divisor: Int) =
+  MapTilePosition(x / divisor, y / divisor)
+
+operator fun MapTilePosition.plus(addend: Int) =
+  MapTilePosition(x + addend, y + addend)
+
+
+fun MapTilePosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
+  MapChunkPosition(
+    floor(x.toDouble() / chunkSize.toDouble()).toInt(),
+    floor(y.toDouble() / chunkSize.toDouble()).toInt(),
+  )
+
+
+fun MapEntityPosition.toMapTilePosition() =
+  MapChunkPosition(floor(x).toInt(), floor(y).toInt())
+
+fun MapEntityPosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
+  MapChunkPosition(
+    floor(x / chunkSize.toDouble()).toInt(),
+    floor(y / chunkSize.toDouble()).toInt(),
+  )
+
+
+operator fun MapChunkPosition.times(factor: Int) =
+  MapChunkPosition(x * factor, y * factor)
+
+operator fun MapChunkPosition.plus(addend: Int) =
+  MapChunkPosition(x + addend, y + addend)
+
+operator fun MapChunkPosition.minus(subtrahend: Int) =
+  MapChunkPosition(x - subtrahend, y - subtrahend)
+
+
+fun MapChunkPosition.leftTopTile(chunkSize: Int = MAP_CHUNK_SIZE): MapTilePosition =
+  MapTilePosition(
+    x * chunkSize,
+    y * chunkSize,
+  )
+
+//val MapChunkPosition.rightBottomTile: MapTilePosition
+//  get() = leftTopTile + (MAP_CHUNK_SIZE - 1)
+//
+//
+//val MapChunkPosition.boundingBox: MapBoundingBox
+//  get() = MapBoundingBox(leftTopTile, rightBottomTile)

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/events/kafkatorioPacket.kt

@@ -0,0 +1,68 @@
+package dev.adamko.kafkatorio.schema.events
+
+import dev.adamko.kafkatorio.schema.SerializerProvider
+import dev.adamko.kafkatorio.schema.SerializerProviderDeserializer
+import dev.adamko.kafkatorio.schema.common.Tick
+import kotlin.reflect.KClass
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+import kotlinx.serialization.KSerializer
+import kotlinx.serialization.Serializable
+
+
+//@Serializable
+//data class KafkatorioPacket(
+//  val modVersion: String,
+//  val tick: Tick,
+//  val data: KafkatorioPacketData,
+//)
+
+
+@Serializable(with = KafkatorioPacketData.JsonSerializer::class)
+sealed class KafkatorioPacketData {
+  abstract val updateType: KafkatorioPacketDataType
+
+  object JsonSerializer : SerializerProviderDeserializer<KafkatorioPacketData>(
+    KafkatorioPacketData::class,
+    KafkatorioPacketData::updateType.name,
+    { json -> KafkatorioPacketDataType.entries.firstOrNull { it.name == json } }
+  )
+}
+
+
+/** Discriminator for [KafkatorioPacketData] */
+@Serializable
+enum class KafkatorioPacketDataType(
+  override val serializer: KSerializer<out KafkatorioPacketData>
+) : SerializerProvider<KafkatorioPacketData> {
+
+  // non-keyed
+  CONFIG(ConfigurationUpdate.serializer()),
+  CONSOLE_CHAT(ConsoleChatUpdate.serializer()),
+  CONSOLE_COMMAND(ConsoleCommandUpdate.serializer()),
+  PROTOTYPES(PrototypesUpdate.serializer()),
+  SURFACE(SurfaceUpdate.serializer()),
+
+  // keyed
+  ENTITY(EntityUpdate.serializer()),
+  MAP_CHUNK(MapChunkUpdate.serializer()),
+  PLAYER(PlayerUpdate.serializer()),
+  ;
+
+  companion object {
+    val entries: Set<KafkatorioPacketDataType> = values().toSet()
+  }
+}
+
+//inline fun <reified T: KafkatorioPacketData> KafkatorioPacketDataType.isType (): KClass<out T> {
+//  return when (this) {
+//    KafkatorioPacketDataType.CONFIG          -> ConfigurationUpdate::class
+//    KafkatorioPacketDataType.CONSOLE_CHAT    -> ConsoleChatUpdate::class
+//    KafkatorioPacketDataType.CONSOLE_COMMAND -> ConsoleCommandUpdate::class
+//    KafkatorioPacketDataType.PROTOTYPES      -> PrototypesUpdate::class
+//    KafkatorioPacketDataType.SURFACE         -> SurfaceUpdate::class
+//    KafkatorioPacketDataType.ENTITY          -> EntityUpdate::class
+//    KafkatorioPacketDataType.MAP_CHUNK       -> MapChunkUpdate::class
+//    KafkatorioPacketDataType.PLAYER          -> PlayerUpdate::class
+//  }
+//}

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/events/kafkatorioPacketInstant.kt

@@ -0,0 +1,95 @@
+package dev.adamko.kafkatorio.schema.events
+
+import dev.adamko.kafkatorio.schema.common.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
+import kotlinx.serialization.EncodeDefault
+import kotlinx.serialization.Serializable
+
+
+/** No key, no caching. Must be sent instantly. */
+sealed class KafkatorioInstantPacketData : KafkatorioPacketData()
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class ConfigurationUpdate(
+  val factorioData: ConfigurationUpdateGameData,
+  val allMods: List<ConfigurationUpdateModData>,
+  val modStartupSettingsChange: Boolean,
+  val migrationApplied: Boolean,
+) : KafkatorioInstantPacketData() {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.CONFIG
+
+  @Serializable
+  data class ConfigurationUpdateGameData(
+    val oldVersion: String? = null,
+    val newVersion: String? = null,
+  )
+
+  @Serializable
+  data class ConfigurationUpdateModData(
+    val modName: String,
+    val currentVersion: String? = null,
+    val previousVersion: String? = null,
+  )
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class ConsoleChatUpdate(
+  val authorPlayerIndex: PlayerIndex?,
+  val content: String,
+) : KafkatorioInstantPacketData() {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.CONSOLE_CHAT
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class ConsoleCommandUpdate(
+  val authorPlayerIndex: PlayerIndex?,
+  val command: String,
+  val parameters: String,
+) : KafkatorioInstantPacketData() {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.CONSOLE_COMMAND
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class PrototypesUpdate(
+  val prototypes: List<FactorioPrototype>,
+) : KafkatorioInstantPacketData() {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.PROTOTYPES
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class SurfaceUpdate(
+  val index: SurfaceIndex,
+  val daytime: Double,
+  val name: String,
+) : KafkatorioInstantPacketData() {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.SURFACE
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/events/kafkatorioPacketKeyed.kt

@@ -0,0 +1,135 @@
+package dev.adamko.kafkatorio.schema.events
+
+import dev.adamko.kafkatorio.schema.common.Colour
+import dev.adamko.kafkatorio.schema.common.EntityIdentifiers
+import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
+import dev.adamko.kafkatorio.schema.common.ForceIndex
+import dev.adamko.kafkatorio.schema.common.MapChunkPosition
+import dev.adamko.kafkatorio.schema.common.MapEntityPosition
+import dev.adamko.kafkatorio.schema.common.MapTileDictionary
+import dev.adamko.kafkatorio.schema.common.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.PrototypeName
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.common.Tick
+import dev.adamko.kafkatorio.schema.common.UnitNumber
+import kotlinx.serialization.EncodeDefault
+import kotlinx.serialization.Serializable
+
+
+sealed interface KafkatorioKeyedPacketKey
+
+
+/** Has a key, so it can be debounced */
+sealed class KafkatorioKeyedPacketData : KafkatorioPacketData() {
+  /** Count how many events this packet is aggregating data from */
+  abstract val eventCounts: Map<String, UInt>?
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+sealed interface EntityUpdateKey : EntityIdentifiers, KafkatorioKeyedPacketKey {
+  /** A [unitNumber] is required for caching */
+  override val unitNumber: UnitNumber
+  override val name: String
+  override val protoType: String
+}
+
+
+@Serializable
+data class EntityUpdate(
+  override val unitNumber: UnitNumber,
+  override val name: String,
+  override val protoType: String,
+
+  override val eventCounts: Map<String, UInt>? = null,
+
+  val chunkPosition: MapChunkPosition? = null,
+  val graphicsVariation: UByte? = null,
+  val health: Float? = null,
+  val isActive: Boolean? = null,
+  val isRotatable: Boolean? = null,
+  val lastUser: UInt? = null,
+  val localisedDescription: String? = null,
+  val localisedName: String? = null,
+  val prototype: PrototypeName? = null,
+) : KafkatorioKeyedPacketData(), EntityUpdateKey {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.ENTITY
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+sealed interface MapChunkUpdateKey : KafkatorioKeyedPacketKey {
+  val chunkPosition: MapChunkPosition
+  val surfaceIndex: SurfaceIndex
+}
+
+
+@Serializable
+data class MapChunkUpdate(
+  override val chunkPosition: MapChunkPosition,
+  override val surfaceIndex: SurfaceIndex,
+
+  override val eventCounts: Map<String, UInt>? = null,
+
+  val player: PlayerIndex? = null,
+  val robot: EntityIdentifiersData? = null,
+  val force: ForceIndex? = null,
+  /** updated tiles - might be partial and not all tiles in the chunk */
+  val tileDictionary: MapTileDictionary? = null,
+  val isDeleted: Boolean? = null,
+) : KafkatorioKeyedPacketData(), MapChunkUpdateKey {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.MAP_CHUNK
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+sealed interface PlayerUpdateKey : KafkatorioKeyedPacketKey {
+  val index: PlayerIndex
+}
+
+
+@Serializable
+data class PlayerUpdate(
+  override val index: PlayerIndex,
+
+  override val eventCounts: Map<String, UInt>? = null,
+
+  val characterUnitNumber: UnitNumber? = null,
+  val chatColour: Colour? = null,
+  val colour: Colour? = null,
+  val name: String? = null,
+
+  val afkTime: Tick? = null,
+  val ticksToRespawn: Tick? = null,
+  val forceIndex: ForceIndex? = null,
+  val isAdmin: Boolean? = null,
+  val isConnected: Boolean? = null,
+  val isShowOnMap: Boolean? = null,
+  val isSpectator: Boolean? = null,
+  val lastOnline: Tick? = null,
+  val onlineTime: Tick? = null,
+  val position: MapEntityPosition? = null,
+  val surfaceIndex: SurfaceIndex? = null,
+  val tag: String? = null,
+  val diedCause: EntityIdentifiersData? = null,
+
+  val bannedReason: String? = null,
+  val kickedReason: String? = null,
+  val disconnectReason: String? = null,
+  /** `true` when a player is removed (deleted) from the game */
+  val isRemoved: Boolean? = null,
+) : KafkatorioKeyedPacketData(), PlayerUpdateKey {
+  @EncodeDefault
+  override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.PLAYER
+}
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/json.kt

@@ -1,13 +1,19 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema
 
+import kotlin.reflect.KClass
+import kotlinx.serialization.DeserializationStrategy
 import kotlinx.serialization.KSerializer
 import kotlinx.serialization.builtins.ListSerializer
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonContentPolymorphicSerializer
 import kotlinx.serialization.json.JsonElement
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.JsonTransformingSerializer
 import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.contentOrNull
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
 import kotlinx.serialization.modules.SerializersModule
 
 
@@ -44,3 +50,28 @@ class FactorioJsonListSerializer<T>(dataSerializer: KSerializer<T>) :
   }
 
 }
+
+
+interface SerializerProvider<T : Any> {
+  val serializer: KSerializer<out T>
+}
+
+open class SerializerProviderDeserializer<T : Any>(
+  private val baseClass: KClass<T>,
+  private val key: String,
+  private val selector: (String) -> SerializerProvider<out T>?,
+) : JsonContentPolymorphicSerializer<T>(baseClass) {
+
+  override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out T> {
+
+    val type = element
+      .jsonObject[key]
+      ?.jsonPrimitive
+      ?.contentOrNull
+      ?.let { json -> selector(json) }
+
+    requireNotNull(type) { "Unknown ${baseClass.simpleName} $key: $element" }
+
+    return type.serializer
+  }
+}

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/prototypes/factorioPrototypes.kt

@@ -1,9 +1,14 @@
-package dev.adamko.kafkatorio.events.schema
+package dev.adamko.kafkatorio.schema.prototypes
 
-import dev.adamko.kafkatorio.events.schema.FactorioPrototype.PrototypeObjectName
+import dev.adamko.kafkatorio.schema.SerializerProvider
+import dev.adamko.kafkatorio.schema.SerializerProviderDeserializer
+import dev.adamko.kafkatorio.schema.common.Colour
+import dev.adamko.kafkatorio.schema.common.PrototypeName
+import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype.PrototypeObjectName
 import kotlinx.serialization.Contextual
 import kotlinx.serialization.DeserializationStrategy
 import kotlinx.serialization.EncodeDefault
+import kotlinx.serialization.KSerializer
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.JsonContentPolymorphicSerializer
 import kotlinx.serialization.json.JsonElement
@@ -12,46 +17,30 @@ import kotlinx.serialization.json.jsonObject
 import kotlinx.serialization.json.jsonPrimitive
 
 
-
+@Serializable(with = FactorioPrototype.JsonSerializer::class)
-@Serializable(with = FactorioPrototypeJsonSerializer::class)
 sealed class FactorioPrototype {
   @EncodeDefault
   abstract val prototypeObjectName: PrototypeObjectName
 
-  enum class PrototypeObjectName {
+  enum class PrototypeObjectName(
-    LuaTilePrototype,
+    override val serializer: KSerializer<out FactorioPrototype>
+  ) : SerializerProvider<FactorioPrototype> {
+    LuaTilePrototype(MapTilePrototype.serializer()),
     ;
 
     companion object {
-      val values: List<PrototypeObjectName> = values().toList()
+      val entries: Set<PrototypeObjectName> = values().toSet()
     }
   }
+
+  object JsonSerializer : SerializerProviderDeserializer<FactorioPrototype>(
+    FactorioPrototype::class,
+    FactorioPrototype::prototypeObjectName.name,
+    { json -> PrototypeObjectName.entries.firstOrNull { it.name == json } }
+  )
 }
 
 
-object FactorioPrototypeJsonSerializer : JsonContentPolymorphicSerializer<FactorioPrototype>(
-  FactorioPrototype::class
-) {
-  private val key = FactorioPrototype::prototypeObjectName.name
-
-  override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out FactorioPrototype> {
-
-    val type = element
-      .jsonObject[key]
-      ?.jsonPrimitive
-      ?.contentOrNull
-      ?.let { json ->
-        PrototypeObjectName.values.firstOrNull { it.name == json }
-      }
-
-    requireNotNull(type) { "Unknown FactorioPrototype ${key}: $element" }
-
-    return when (type) {
-      PrototypeObjectName.LuaTilePrototype -> MapTilePrototype.serializer()
-    }
-  }
-}
-
 
 @Serializable
 data class MapTilePrototype(

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/prototypes/factorioPrototypes2.kt

@@ -0,0 +1,23 @@
+package dev.adamko.kafkatorio.schema.prototypes
+
+import dev.adamko.kafkatorio.schema.common.Colour
+import dev.adamko.kafkatorio.schema.common.PrototypeName
+import kotlinx.serialization.Contextual
+import kotlinx.serialization.Serializable
+
+
+@Serializable
+sealed class FactorioPrototype2 {
+
+  @Serializable
+  data class MapTile(
+    val name: PrototypeName,
+    val layer: UInt,
+    val mapColour: Colour,
+    @Contextual
+    val collisionMasks: List<String>,
+    val order: String,
+    /** Can the tile be mined for resources? */
+    val canBeMined: Boolean,
+  ) : FactorioPrototype2()
+}

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema2/kafkatorioPacket2.kt

@@ -0,0 +1,16 @@
+package dev.adamko.kafkatorio.schema2
+
+import dev.adamko.kafkatorio.schema.common.Tick
+import kotlinx.serialization.Serializable
+
+
+@Serializable
+data class KafkatorioPacket2(
+  val modVersion: String,
+  val tick: Tick,
+  val data: KafkatorioPacketData2,
+)
+
+
+@Serializable
+sealed class KafkatorioPacketData2

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema2/kafkatorioPacketInstant2.kt

@@ -0,0 +1,70 @@
+package dev.adamko.kafkatorio.schema2
+
+import dev.adamko.kafkatorio.schema.common.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
+import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
+import kotlinx.serialization.Serializable
+
+
+/** No key, no caching. Must be sent instantly. */
+@Serializable
+sealed class KafkatorioInstantPacketData2 : KafkatorioPacketData2()
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class ConfigurationUpdate(
+  val factorioData: ConfigurationUpdateGameData,
+  val allMods: List<ConfigurationUpdateModData>,
+  val modStartupSettingsChange: Boolean,
+  val migrationApplied: Boolean,
+) : KafkatorioInstantPacketData2() {
+
+  @Serializable
+  data class ConfigurationUpdateGameData(
+    val oldVersion: String? = null,
+    val newVersion: String? = null,
+  )
+
+  @Serializable
+  data class ConfigurationUpdateModData(
+    val modName: String,
+    val currentVersion: String? = null,
+    val previousVersion: String? = null,
+  )
+}
+
+
+@Serializable
+data class ConsoleChatUpdate(
+  val authorPlayerIndex: PlayerIndex?,
+  val content: String,
+) : KafkatorioInstantPacketData2()
+
+
+@Serializable
+data class ConsoleCommandUpdate(
+  val authorPlayerIndex: PlayerIndex?,
+  val command: String,
+  val parameters: String,
+) : KafkatorioInstantPacketData2()
+
+
+@Serializable
+data class PrototypesUpdate(
+  val prototypes: List<FactorioPrototype2>,
+) : KafkatorioInstantPacketData2()
+
+
+@Serializable
+data class SurfaceUpdate(
+  val index: SurfaceIndex,
+  val daytime: Double,
+  val name: String,
+) : KafkatorioInstantPacketData2()
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema2/kafkatorioPacketKeyed2.kt

@@ -0,0 +1,126 @@
+package dev.adamko.kafkatorio.schema2
+
+import dev.adamko.kafkatorio.schema.common.Colour
+import dev.adamko.kafkatorio.schema.common.EntityIdentifiers
+import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
+import dev.adamko.kafkatorio.schema.common.ForceIndex
+import dev.adamko.kafkatorio.schema.common.MapChunkPosition
+import dev.adamko.kafkatorio.schema.common.MapEntityPosition
+import dev.adamko.kafkatorio.schema.common.MapTileDictionary
+import dev.adamko.kafkatorio.schema.common.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.PrototypeName
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.common.Tick
+import dev.adamko.kafkatorio.schema.common.UnitNumber
+import kotlinx.serialization.Serializable
+
+
+/** Has a key, so it can be debounced */
+@Serializable
+sealed class KafkatorioKeyedPacketData2 : KafkatorioPacketData2() {
+  /** Count how many events this packet is aggregating data from */
+  abstract val key: KafkatorioKeyedPacketKey2
+  abstract val eventCounts: Map<String, UInt>?
+}
+
+@Serializable
+sealed class KafkatorioKeyedPacketKey2
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class EntityUpdateKey(
+  /** While normally optional, here a [unitNumber] is required for caching */
+  override val unitNumber: UnitNumber,
+  override val name: String,
+  override val protoType: String,
+) : EntityIdentifiers, KafkatorioKeyedPacketKey2()
+
+
+@Serializable
+data class EntityUpdate(
+  override val key: EntityUpdateKey,
+
+  override val eventCounts: Map<String, UInt>? = null,
+
+  val chunkPosition: MapChunkPosition? = null,
+  val graphicsVariation: UByte? = null,
+  val health: Float? = null,
+  val isActive: Boolean? = null,
+  val isRotatable: Boolean? = null,
+  val lastUser: UInt? = null,
+  val localisedDescription: String? = null,
+  val localisedName: String? = null,
+  val prototype: PrototypeName? = null,
+) : KafkatorioKeyedPacketData2()
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+
+@Serializable
+data class MapChunkUpdateKey(
+  val chunkPosition: MapChunkPosition,
+  val surfaceIndex: SurfaceIndex,
+) : KafkatorioKeyedPacketKey2()
+
+
+@Serializable
+data class MapChunkUpdate(
+  override val key: MapChunkUpdateKey,
+
+  override val eventCounts: Map<String, UInt>? = null,
+
+  val player: PlayerIndex? = null,
+  val robot: EntityIdentifiersData? = null,
+  val force: ForceIndex? = null,
+  /** updated tiles - might be partial and not all tiles in the chunk */
+  val tileDictionary: MapTileDictionary? = null,
+  val isDeleted: Boolean? = null,
+) : KafkatorioKeyedPacketData2()
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */
+
+@Serializable
+data class PlayerUpdateKey(
+  val index: PlayerIndex,
+) : KafkatorioKeyedPacketKey2()
+
+
+@Serializable
+data class PlayerUpdate(
+  override val key: PlayerUpdateKey,
+
+  override val eventCounts: Map<String, UInt>? = null,
+
+  val characterUnitNumber: UnitNumber? = null,
+  val chatColour: Colour? = null,
+  val colour: Colour? = null,
+  val name: String? = null,
+
+  val afkTime: Tick? = null,
+  val ticksToRespawn: Tick? = null,
+  val forceIndex: ForceIndex? = null,
+  val isAdmin: Boolean? = null,
+  val isConnected: Boolean? = null,
+  val isShowOnMap: Boolean? = null,
+  val isSpectator: Boolean? = null,
+  val lastOnline: Tick? = null,
+  val onlineTime: Tick? = null,
+  val position: MapEntityPosition? = null,
+  val surfaceIndex: SurfaceIndex? = null,
+  val tag: String? = null,
+  val diedCause: EntityIdentifiersData? = null,
+
+  val bannedReason: String? = null,
+  val kickedReason: String? = null,
+  val disconnectReason: String? = null,
+  /** `true` when a player is removed (deleted) from the game */
+  val isRemoved: Boolean? = null,
+) : KafkatorioKeyedPacketData2()
+
+
+/*  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  ***  *** */

modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/ColourHexTest.kt

@@ -1,7 +1,10 @@
 package dev.adamko.kafkatorio.events.schema
 
-import dev.adamko.kafkatorio.events.schema.converters.toHex
+import dev.adamko.kafkatorio.schema.common.Colour
-import dev.adamko.kafkatorio.events.schema.converters.toPercentile
+import dev.adamko.kafkatorio.schema.common.ColourHex
+import dev.adamko.kafkatorio.schema.common.toHex
+import dev.adamko.kafkatorio.schema.common.toPercentile
+import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
 import io.kotest.assertions.withClue
 import io.kotest.core.spec.style.BehaviorSpec
 import io.kotest.matchers.floats.shouldBeWithinPercentageOf
@@ -81,6 +84,4 @@ class ColourHexTest : BehaviorSpec({
 
     }
   }
-
-
 })

modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/FactorioEventTest.kt

@@ -1,342 +0,0 @@
-package dev.adamko.kafkatorio.events.schema
-
-import io.kotest.assertions.json.shouldEqualJson
-import io.kotest.core.spec.style.FunSpec
-import io.kotest.matchers.shouldBe
-import kotlinx.serialization.decodeFromString
-import kotlinx.serialization.encodeToString
-
-class FactorioEventTest : FunSpec({
-
-//  context("Given: json on_player_joined_game") {
-//
-//    // language=JSON
-//    val json = """
-//       {
-//         "data": {
-//           "objectName": "LuaPlayer",
-//           "force": 1,
-//           "surface": 1,
-//           "name": "fredthedeadhead",
-//           "characterUnitNumber": 1,
-//           "associatedCharactersUnitNumbers": {},
-//           "position": {
-//             "x": 30.6,
-//             "y": 84.1
-//           },
-//           "colour": {
-//             "red": 0.86,
-//             "green": 0.5,
-//             "blue": 0.12,
-//             "alpha": 0.5
-//           },
-//           "chatColour": {
-//             "red": 1,
-//             "green": 0.62,
-//             "blue": 0.25
-//           },
-//           "lastOnline": 2287061
-//         },
-//         "packetType": "EVENT",
-//         "eventType": "on_player_joined_game",
-//         "modVersion": "0.2.1",
-//         "tick": 2287072
-//       }
-//    """.trimIndent()
-//    context("When: decoded") {
-//      val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
-//
-//      test("Then: parse") {
-//
-//        val expected = FactorioEvent(
-//          data = PlayerData(
-//            name = "fredthedeadhead",
-//            surface = SurfaceIndex(1u),
-//            force = ForceIndex(1u),
-//            characterUnitNumber = UnitNumber(1u),
-//            associatedCharactersUnitNumbers = emptyList(),
-//            position = MapEntityPosition(
-//              x = 30.6,
-//              y = 84.1,
-//            ),
-//            colour = Colour(
-//              red = 0.86f,
-//              green = 0.5f,
-//              blue = 0.12f,
-//              alpha = 0.5f
-//            ),
-//            chatColour = Colour(
-//              red = 1f,
-//              green = 0.62f,
-//              blue = 0.25f,
-//              alpha = 1f
-//            ),
-//            lastOnline = Tick(2287061u)
-//          ),
-//          eventType = "on_player_joined_game",
-//          modVersion = "0.2.1",
-//          tick = Tick(2287072u)
-//        )
-//
-//        actual shouldBe expected
-//
-//        expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaPlayer
-//      }
-//
-//      test("Then: expect encode equals json") {
-//        val encoded = jsonMapperKafkatorio.encodeToString(actual)
-//        encoded.shouldEqualJson(json)
-//      }
-//    }
-//  }
-//
-//  context("Given: json LuaSurface") {
-//    // language=JSON
-//    val json = """
-//      {
-//        "data": {
-//          "objectName": "LuaSurface",
-//          "name": "nauvis",
-//          "index": 1,
-//          "daytime": 0.745
-//        },
-//        "packetType": "EVENT",
-//        "eventType": "on_tick",
-//        "modVersion": "0.2.1",
-//        "tick": 2301240
-//      }
-//    """.trimIndent()
-//
-//    context("When: decoded") {
-//      val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
-//
-//      test("should equal obj") {
-//
-//        val expected = FactorioEvent(
-//          data = SurfaceData(
-//            name = "nauvis",
-//            index = SurfaceIndex(1u),
-//            daytime = 0.745
-//          ),
-//          eventType = "on_tick",
-//          modVersion = "0.2.1",
-//          tick = Tick(2301240u)
-//        )
-//
-//        actual shouldBe expected
-//
-//        expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaSurface
-//      }
-//
-//      test("Then: expect encode equals json") {
-//        val encoded = jsonMapperKafkatorio.encodeToString(actual)
-//        encoded.shouldEqualJson(json)
-//      }
-//    }
-//  }
-//
-//  context("Given: LuaEntity on_player_changed_position") {
-//    // language=JSON
-//    val json = """
-//      {
-//        "data": {
-//          "objectName": "LuaEntity",
-//          "name": "character",
-//          "type": "character",
-//          "active": true,
-//          "health": 250,
-//          "healthRatio": 1,
-//          "surface": 1,
-//          "force": 1,
-//          "unitNumber": 1,
-//          "position": {
-//            "x": 37.5,
-//            "y": 81.1
-//          }
-//        },
-//        "packetType": "EVENT",
-//        "eventType": "on_player_changed_position",
-//        "modVersion": "0.2.1",
-//        "tick": 2301231
-//      }
-//    """.trimIndent()
-//    context("When: decoded") {
-//      val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
-//
-//      test("parse") {
-//
-//        val expected = FactorioEvent(
-//          data = EntityData(
-//            name = "character",
-//            type = "character",
-//            active = true,
-//            health = 250.0,
-//            healthRatio = 1.0,
-//            surface = SurfaceIndex(1u),
-//            unitNumber = UnitNumber(1u),
-//            force = ForceIndex(1u),
-//            playerIndex = null,
-//            position = MapEntityPosition(
-//              x = 37.5,
-//              y = 81.1,
-//            )
-//          ),
-//          eventType = "on_player_changed_position",
-//          modVersion = "0.2.1",
-//          tick = Tick(2301231u)
-//        )
-//
-//        actual shouldBe expected
-//
-//        expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaEntity
-//      }
-//
-//      test("Then: expect encode equals json") {
-//        val encoded = jsonMapperKafkatorio.encodeToString(actual)
-//        encoded.shouldEqualJson(json)
-//      }
-//    }
-//
-//  }
-//
-//  context("Given: LuaTiles on_player_built_tile") {
-//    // language=JSON
-//    val json = """
-//      {
-//        "data": {
-//          "objectName": "LuaTiles",
-//          "surfaceIndex": 1,
-//          "tiles": [
-//            {
-//              "y": 10,
-//              "x": -65,
-//              "proto": "refined-concrete"
-//            },
-//            {
-//              "y": 11,
-//              "x": -65,
-//              "proto": "red-earth"
-//            },
-//            {
-//              "y": 10,
-//              "x": -64,
-//              "proto": "blue-cheese"
-//            },
-//            {
-//              "y": 11,
-//              "x": -64,
-//              "proto": "refined-concrete"
-//            }
-//          ]
-//        },
-//        "packetType": "EVENT",
-//        "eventType": "on_player_built_tile",
-//        "modVersion": "0.2.2",
-//        "tick": 2429929
-//      }
-//    """.trimIndent()
-//    context("When: decoded") {
-//      val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
-//
-//      test("parse") {
-//
-//        val expected = FactorioEvent(
-//          data = MapTiles(
-//            surfaceIndex = SurfaceIndex(1u),
-//            tiles = listOf(
-//              MapTile(
-//                proto = PrototypeName("refined-concrete"),
-//                x = -65,
-//                y = 10,
-//              ),
-//              MapTile(
-//                proto = PrototypeName("red-earth"),
-//                x = -65,
-//                y = 11,
-//              ),
-//              MapTile(
-//                proto = PrototypeName("blue-cheese"),
-//                x = -64,
-//                y = 10,
-//              ),
-//              MapTile(
-//                proto = PrototypeName("refined-concrete"),
-//                x = -64,
-//                y = 11,
-//              ),
-//            )
-//          ),
-//          eventType = "on_player_built_tile",
-//          modVersion = "0.2.2",
-//          tick = Tick(2429929u)
-//        )
-//
-//        actual shouldBe expected
-//
-//        expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaTiles
-//      }
-//
-//      test("Then: expect encode equals json") {
-//        val encoded = jsonMapperKafkatorio.encodeToString(actual)
-//        encoded.shouldEqualJson(json)
-//      }
-//    }
-//  }
-//
-//  context("Given: packet FactorioConfigurationUpdate") {
-//    // language=JSON
-//    val json = """
-//               {
-//                 "modVersion": "0.2.8",
-//                 "packetType": "CONFIG",
-//                 "tick": 534,
-//                 "allMods": [
-//                   {
-//                     "modName": "kafkatorio-events",
-//                     "currentVersion": "0.2.8",
-//                     "previousVersion": "0.2.7"
-//                   },
-//                   {
-//                     "modName": "base",
-//                     "currentVersion": "1.1.53"
-//                   }
-//                 ],
-//                 "factorioData": {}
-//               }
-//               """.trimIndent()
-//    context("When: decoded") {
-//      val actual: FactorioConfigurationUpdate = jsonMapperKafkatorio.decodeFromString(json)
-//
-//      test("parse") {
-//
-//        val expected = FactorioConfigurationUpdate(
-//          modVersion = "0.2.8",
-//          tick = Tick(534u),
-//          factorioData = FactorioGameDataUpdate(null, null),
-//          allMods = listOf(
-//            FactorioModInfo(
-//              modName = "kafkatorio-events",
-//              currentVersion = "0.2.8",
-//              previousVersion = "0.2.7"
-//            ),
-//            FactorioModInfo(
-//              modName = "base",
-//              currentVersion = "1.1.53",
-//              previousVersion = null
-//            ),
-//          )
-//        )
-//
-//        actual shouldBe expected
-//
-//        expected.packetType shouldBe KafkatorioPacket.PacketType.CONFIG
-//      }
-//
-//      test("Then: expect encode equals json") {
-//        val encoded = jsonMapperKafkatorio.encodeToString(actual)
-//        encoded.shouldEqualJson(json)
-//      }
-//    }
-//
-//  }
-})

modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/MapChunkUpdateTest.kt

@@ -0,0 +1,93 @@
+package dev.adamko.kafkatorio.events.schema
+
+import dev.adamko.kafkatorio.schema.common.MapChunkPosition
+import dev.adamko.kafkatorio.schema.common.MapTileDictionary
+import dev.adamko.kafkatorio.schema.common.PrototypeName
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.common.Tick
+import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
+import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
+import dev.adamko.kafkatorio.schema2.MapChunkUpdate
+import dev.adamko.kafkatorio.schema2.MapChunkUpdateKey
+import io.kotest.core.spec.style.FunSpec
+import io.kotest.matchers.shouldBe
+import kotlin.math.exp
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+
+class MapChunkUpdateTest : FunSpec({
+
+  context("MapChunkUpdate update") {
+    //language=json
+    val json = """
+{
+  "data": {
+    "type": "dev.adamko.kafkatorio.schema2.MapChunkUpdate",
+    "key": {
+      "surfaceIndex": 1,
+      "chunkPosition": [
+        -6,
+        -3
+      ]
+    },
+    "tileDictionary": {
+      "tilesXY": {
+        "-192": {
+          "-65": 2
+        },
+        "161": {
+          "-96": 1,
+          "-95": 2,
+          "94": 2,
+          "93": 32
+        }
+      },
+      "protos": {
+        "water": 1,
+        "grass-4": 2,
+        "grass-3": 32
+      }
+    },
+    "eventCounts": {
+      "on_chunk_generated": 1
+    }
+  },
+  "modVersion": "0.4.0",
+  "tick": 36
+}
+    """.trimIndent()
+
+    val expected = KafkatorioPacket2(
+      tick = Tick(36u),
+      modVersion = "0.4.0",
+      data = MapChunkUpdate(
+        key = MapChunkUpdateKey(
+          MapChunkPosition(-6, -3),
+          SurfaceIndex(1u),
+        ),
+        eventCounts = mapOf("on_chunk_generated" to 1u),
+        tileDictionary = MapTileDictionary(
+          tilesXY = mapOf(
+            "-192" to mapOf("-65" to MapTileDictionary.PrototypeKey(2)),
+            "161" to mapOf(
+              "-96" to MapTileDictionary.PrototypeKey(1),
+              "-95" to MapTileDictionary.PrototypeKey(2),
+              "94" to MapTileDictionary.PrototypeKey(2),
+              "93" to MapTileDictionary.PrototypeKey(32),
+            ),
+          ),
+          protos = mapOf(
+            PrototypeName("water") to MapTileDictionary.PrototypeKey(1),
+            PrototypeName("grass-4") to MapTileDictionary.PrototypeKey(2),
+            PrototypeName("grass-3") to MapTileDictionary.PrototypeKey(32),
+          )
+        )
+      )
+    )
+
+    test("decode") {
+      val packet: KafkatorioPacket2 = jsonMapperKafkatorio.decodeFromString(json)
+      packet shouldBe expected
+    }
+  }
+})

modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTileDictionaryTest.kt

@@ -0,0 +1,42 @@
+package dev.adamko.kafkatorio.schema.common
+
+import io.kotest.core.spec.style.FunSpec
+import io.kotest.matchers.collections.shouldContainAll
+
+class MapTileDictionaryTest : FunSpec({
+
+  context("MapTileDictionary") {
+    val actual = MapTileDictionary(
+      tilesXY = mapOf(
+        "-192" to mapOf("-65" to MapTileDictionary.PrototypeKey(2)),
+        "161" to mapOf(
+          "-96" to MapTileDictionary.PrototypeKey(1),
+          "-95" to MapTileDictionary.PrototypeKey(2),
+          "94" to MapTileDictionary.PrototypeKey(2),
+          "93" to MapTileDictionary.PrototypeKey(32),
+        ),
+
+        // unknown prototype
+        "44" to mapOf("1" to MapTileDictionary.PrototypeKey(44)),
+      ),
+      protos = mapOf(
+        PrototypeName("water") to MapTileDictionary.PrototypeKey(1),
+        PrototypeName("grass-4") to MapTileDictionary.PrototypeKey(2),
+        PrototypeName("grass-3") to MapTileDictionary.PrototypeKey(32),
+
+        PrototypeName("not-used") to MapTileDictionary.PrototypeKey(99),
+      )
+    )
+
+    test("toMapTileList") {
+      actual.toMapTileList() shouldContainAll listOf(
+        MapTile(-192, -65, PrototypeName("grass-4")),
+
+        MapTile(161, -96, PrototypeName("water")),
+        MapTile(161, -95, PrototypeName("grass-4")),
+        MapTile(161, 94, PrototypeName("grass-4")),
+        MapTile(161, 93, PrototypeName("grass-3")),
+      )
+    }
+  }
+})

modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts.kt

@@ -1,5 +1,10 @@
 package dev.adamko.kafkatorio.events.schema
 
+import dev.adamko.kafkatorio.schema.events.KafkatorioInstantPacketData
+import dev.adamko.kafkatorio.schema.events.KafkatorioKeyedPacketData
+import dev.adamko.kafkatorio.schema.events.KafkatorioKeyedPacketKey
+//import dev.adamko.kafkatorio.schema.events.KafkatorioPacket
+import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
 import kotlin.io.path.Path
 import kotlin.io.path.createDirectories
 import me.ntrrgc.tsGenerator.TypeScriptGenerator
@@ -10,22 +15,20 @@ import org.intellij.lang.annotations.Language
  */
 fun main(args: Array<String>) {
 
-  val gen =
+  val gen = TypeScriptGenerator(
-    TypeScriptGenerator(
     rootClasses = buildSet {
-        addAll(KafkatorioPacket::class.sealedSubclasses)
+//      add(KafkatorioPacket::class)
+//      addAll(KafkatorioPacket::class.sealedSubclasses)
 
-        addAll(FactorioObjectData::class.sealedSubclasses)
+      add(KafkatorioInstantPacketData::class)
+      addAll(KafkatorioInstantPacketData::class.sealedSubclasses)
 
+      add(KafkatorioKeyedPacketData::class)
+      addAll(KafkatorioKeyedPacketData::class.sealedSubclasses)
+      addAll(KafkatorioKeyedPacketKey::class.sealedSubclasses)
+
+      add(FactorioPrototype::class)
       addAll(FactorioPrototype::class.sealedSubclasses)
-        add(FactorioPrototypes::class)
-        add(FactorioConfigurationUpdate::class)
-
-        add(FactorioEvent::class)
-
-        add(FactorioEventUpdatePacket::class)
-        addAll(FactorioEventUpdate::class.sealedSubclasses)
-        addAll(FactorioEventUpdateKey::class.sealedSubclasses)
     },
     mappings = mapOf(
       // builtin Factorio numeric types > `typed-factorio/generated/builtin-types.d.ts`
@@ -57,7 +60,7 @@ fun main(args: Array<String>) {
         .forEach { (defFile, def) ->
           val outFile = outPath.resolve(defFile).toFile()
           outFile.printWriter().use {
-            it.println(header)
+            it.println(kt2tsHeader)
             it.print(def)
           }
           println("Generated file: $outFile")
@@ -67,6 +70,7 @@ fun main(args: Array<String>) {
   }
 }
 
+
 /** Split the TypeScript interfaces into different files */
 private fun splitDefinitions(definitions: String): Map<String, String> {
   return definitions
@@ -74,51 +78,64 @@ private fun splitDefinitions(definitions: String): Map<String, String> {
     .groupBy { def ->
 
       when {
+        def isInterface "KafkatorioPacket"
+            || def extendsAny listOf("KafkatorioPacket")
+            || def isType "KafkatorioPacketDataType"
+             -> "kafkatorio-packet.d.ts"
 
-        "interface FactorioObjectData " in def ||
+        def isInterface "KafkatorioInstantPacketData"
-            "type ObjectName " in def ||
+            || def extends "KafkatorioInstantPacketData"
-            def.contains(Regex("interface.+extends.+(FactorioObjectData).*"))
+            || def isType "KafkatorioInstantPacketDataType"
-                                               -> "object-data.d.ts"
 
-        "extends FactorioPrototype " in def ||
+            || def isInterface "ConfigurationUpdateGameData"
-            "interface FactorioPrototypes " in def ||
+            || def isInterface "ConfigurationUpdateModData"
-            "interface FactorioPrototype " in def ||
+             -> "kafkatorio-instant-packet.d.ts"
-            "type PrototypeObjectName " in def -> "prototype.d.ts"
 
-        "extends FactorioConfigurationUpdate " in def ||
+        def isInterface "KafkatorioKeyedPacket"
-            "interface FactorioConfigurationUpdate " in def ||
+            || def isInterface "KafkatorioKeyedPacketData"
-            "FactorioGameDataUpdate " in def ||
+            || def extends "KafkatorioKeyedPacket"
-            "FactorioModInfo " in def          -> "config-update.d.ts"
+            || def isType "KafkatorioKeyedPacketDataType"
 
-        "interface FactorioEventUpdatePacket" in def ||
+            || def isInterface "KafkatorioKeyedPacketKey"
-            "type FactorioEventUpdateType =" in def ||
+             -> "kafkatorio-keyed-packet.d.ts"
-            "interface FactorioEventUpdateKey {" in def ||
+
-            "interface FactorioEventUpdateData {" in def ||
+        def isInterface "FactorioPrototypes"
-            def.contains(
+            || def isInterface "FactorioPrototype"
-              Regex(
+            || def extends "FactorioPrototype"
-                """
+            || def isType "PrototypeObjectName"
-                interface.+extends.+(FactorioEventUpdate|PlayerUpdateKey|PlayerUpdateData).*
+             -> "prototype.d.ts"
-                """.trimIndent()
-              )
-            )
-                                               -> "update.d.ts"
 
         else -> "schema.d.ts"
       }
     }
     .mapValues { (_, def) ->
       def
+        .sorted()
         .joinToString("\n\n", postfix = "\n")
         .replace("    ", "  ")
     }
 }
 
 @Language("TypeScript")
-val header = """
+val kt2tsHeader = """
   |// Generated by TypeScriptGenerator - do not edit this file manually
   |
 """.trimMargin()
 
+private infix fun String.extends(types: String): Boolean =
+  contains(Regex(".+extends.*(${types})"))
+
+private infix fun String.extendsAny(types: List<String>): Boolean =
+  extends(types.joinToString("|") { "$it,".replaceAfterLast(',', "") })
+
+private infix fun String.isInterface(type: String): Boolean =
+  contains(Regex("interface $type.*"))
+
+
+private infix fun String.isType(type: String): Boolean =
+  contains(Regex("type $type.*"))
+
+
 //object PropertyNameTransformer : ClassTransformer {
 //  override fun transformPropertyName(
 //    propertyName: String,

modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts2.kt

@@ -0,0 +1,55 @@
+package dev.adamko.kafkatorio.events.schema
+
+import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
+import dev.adamko.kxstsgen.KxsTsConfig
+import dev.adamko.kxstsgen.KxsTsGenerator
+import dev.adamko.kxstsgen.core.TsDeclaration.TsTypeAlias
+import dev.adamko.kxstsgen.core.TsElementId
+import dev.adamko.kxstsgen.core.TsLiteral
+import dev.adamko.kxstsgen.core.TsTypeRef
+import kotlin.io.path.Path
+import kotlin.io.path.createDirectories
+import kotlinx.serialization.builtins.serializer
+
+
+@ExperimentalUnsignedTypes
+fun main(args: Array<String>) {
+
+  val config = KxsTsConfig(
+//    typeAliasTyping = BrandTyping
+  )
+
+  val gen2 = KxsTsGenerator(config)
+
+  gen2.descriptorOverrides += mapOf(
+//@formatter:off
+    Float .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Float" ), TsTypeRef.Literal(TsLiteral.Custom("float" ), false)),
+    Double.serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Double"), TsTypeRef.Literal(TsLiteral.Custom("double"), false)),
+    Int   .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Int"   ), TsTypeRef.Literal(TsLiteral.Custom("int"   ), false)),
+    Byte  .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Byte"  ), TsTypeRef.Literal(TsLiteral.Custom("int8"  ), false)),
+    UInt  .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.UInt"  ), TsTypeRef.Literal(TsLiteral.Custom("uint"  ), false)),
+    UByte .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.UByte" ), TsTypeRef.Literal(TsLiteral.Custom("uint8" ), false)),
+    UShort.serializer().descriptor to TsTypeAlias(TsElementId("kotlin.UShort"), TsTypeRef.Literal(TsLiteral.Custom("uint16"), false)),
+    ULong .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.ULong" ), TsTypeRef.Literal(TsLiteral.Custom("uint64"), false)),
+//@formatter:on
+  )
+
+  val definitions = gen2.generate(
+    KafkatorioPacket2.serializer(),
+//    KafkatorioKeyedPacketKey2.serializer(),
+  )
+
+
+  when (val filename = args.firstOrNull()) {
+    null -> println(definitions)
+    else -> {
+      val outPath = Path(filename)
+      outPath.createDirectories()
+      val outFile = outPath.resolve("kafkatorio-schema.ts").toFile()
+      outFile.printWriter().use {
+        it.println(kt2tsHeader)
+        it.print(definitions)
+      }
+    }
+  }
+}

modules/events-server/build.gradle.kts

@@ -41,6 +41,9 @@ dependencies {
 
   implementation(libs.scrimage.core)
 
+  implementation(platform(libs.okio.bom))
+  implementation(libs.okio.core)
+
   implementation(libs.kafka.kotkaStreams) {
     isChanging = true
   }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt

@@ -7,9 +7,9 @@ import dev.adamko.kafkatorio.processor.topology.playerUpdatesToWsServer
 import dev.adamko.kafkatorio.processor.topology.saveMapTiles
 import dev.adamko.kafkatorio.processor.topology.splitFactorioServerPacketStream
 import java.time.Duration
+import java.util.Properties
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.cancellation.CancellationException
-import kotlin.time.Duration.Companion.minutes
 import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -31,15 +31,12 @@ internal class KafkatorioTopology(
   override val coroutineContext: CoroutineContext =
     Dispatchers.Default + SupervisorJob() + CoroutineName("KafkatorioTopology")
 
-  companion object {
-    const val TOPIC_SRC_SERVER_LOG = "factorio-server-log"
-    const val TOPIC_GROUPED_MAP_CHUNKS = "kafkatorio.MAP.grouped-map-chunks"
-  }
-
   fun start() {
+    splitPackets()
+
     groupTilesMapChunks()
     saveTiles()
-    splitPackets()
+
     playerUpdates()
   }
 
@@ -56,7 +53,6 @@ internal class KafkatorioTopology(
   }
 
   private fun splitPackets() {
-
     val builder = StreamsBuilder()
     val packets = factorioServerPacketStream(builder)
     splitFactorioServerPacketStream(packets)
@@ -75,12 +71,19 @@ internal class KafkatorioTopology(
     launchTopology("playerUpdates", topology)
   }
 
-  private fun launchTopology(id: String, topology: Topology) {
+  private fun launchTopology(
+    id: String,
+    topology: Topology,
+    config: Map<String, String> = mapOf()
+  ) {
 
-    val props = appProps.kafkaConfig
+    val props: Properties = appProps.kafkaConfig
+
+    config.forEach { (k, v) -> props.setProperty(k, v) }
 
     val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as? String
     props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
+
 //    props.setProperty(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG, "${30.minutes.inWholeMilliseconds}")
 
     val streams = KafkaStreams(topology, props)

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/StreamsExceptionHandler.kt

@@ -3,6 +3,7 @@ package dev.adamko.kafkatorio.processor
 import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler
 import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse
 
+
 class StreamsExceptionHandler(
   val shutdown: (e: Throwable) -> Unit,
 ) : StreamsUncaughtExceptionHandler {

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/KafkatorioKafkaAdmin.kt

@@ -1,9 +1,5 @@
 package dev.adamko.kafkatorio.processor.admin
 
-import dev.adamko.kafkatorio.events.schema.FactorioEventUpdate
-import dev.adamko.kafkatorio.events.schema.FactorioObjectData
-import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
-import dev.adamko.kafkatorio.processor.KafkatorioTopology
 import java.util.concurrent.TimeUnit
 import mu.KotlinLogging
 import org.apache.kafka.clients.admin.Admin
@@ -28,34 +24,9 @@ object KafkatorioKafkaAdmin {
 
     val currentTopics = currentTopics().map { it.name() }
 
-    val kafkatorioTopics = buildSet {
+    val kafkatorioTopics = allTopics() - currentTopics.toSet()
-      KafkatorioPacket.PacketType.values.forEach { packetType ->
-
-        add(KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS)
-        add(KafkatorioTopology.TOPIC_SRC_SERVER_LOG)
-
-        val base = topicNameBase(packetType)
-
-        when (packetType) {
-          KafkatorioPacket.PacketType.EVENT      ->
-            FactorioObjectData.ObjectName.values.forEach { objectName ->
-              add("$base.${objectName.name}")
-            }
-          KafkatorioPacket.PacketType.CONFIG     -> add("$base.FactorioConfigurationUpdate")
-          KafkatorioPacket.PacketType.PROTOTYPES -> add("$base.all")
-          KafkatorioPacket.PacketType.UPDATE     -> {
-            add("$base.all")
-            FactorioEventUpdate.FactorioEventUpdateType.values.forEach { objectName ->
-              add("$base.${objectName.name}")
-            }
-          }
-        }
-
-      }
-    }.minus(currentTopics.toSet())
 
     if (kafkatorioTopics.isNotEmpty()) {
-
       logger.info("Creating ${kafkatorioTopics.size} topics")
       val result = kafkaAdmin.createTopics { kafkatorioTopics }
 
@@ -83,11 +54,14 @@ object KafkatorioKafkaAdmin {
     replicationFactor: Short = 1,
     topicNames: () -> Collection<String>,
   ): CreateTopicsResult =
-    createTopics(topicNames()
+    createTopics(
+      topicNames()
         .distinct()
         .map {
           NewTopic(it, numPartitions, replicationFactor)
-      })
+//            .configs(mapOf("cleanup.policy" to "compact"))
+        }
+    )
 
   private fun currentTopics(): MutableCollection<TopicListing> {
     return kafkaAdmin.listTopics()
@@ -95,6 +69,6 @@ object KafkatorioKafkaAdmin {
       .get()
   }
 
-  fun topicNameBase(packetType: KafkatorioPacket.PacketType): String =
+//  fun topicNameBase(packetType: KafkatorioPacketDataType): String =
-    "kafkatorio.${packetType.name}"
+//    "kafkatorio.${packetType.name}"
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt

@@ -0,0 +1,104 @@
+package dev.adamko.kafkatorio.processor.admin
+
+import dev.adamko.kafkatorio.schema2.ConfigurationUpdate
+import dev.adamko.kafkatorio.schema2.ConsoleChatUpdate
+import dev.adamko.kafkatorio.schema2.ConsoleCommandUpdate
+import dev.adamko.kafkatorio.schema2.EntityUpdate
+import dev.adamko.kafkatorio.schema2.KafkatorioPacketData2
+import dev.adamko.kafkatorio.schema2.MapChunkUpdate
+import dev.adamko.kafkatorio.schema2.PlayerUpdate
+import dev.adamko.kafkatorio.schema2.PrototypesUpdate
+import dev.adamko.kafkatorio.schema2.SurfaceUpdate
+import kotlin.reflect.KClass
+
+
+private const val DOMAIN = "kafkatorio"
+
+
+const val TOPIC_SRC_SERVER_LOG = "$DOMAIN.src.server-log"
+//  "factorio-server-log"
+const val TOPIC_GROUPED_MAP_CHUNKS_STATE = "$DOMAIN.state.map-chunks.grouped"
+
+
+fun allTopics(): Set<String> = buildSet {
+  add(TOPIC_SRC_SERVER_LOG)
+  add(TOPIC_GROUPED_MAP_CHUNKS_STATE)
+
+  addAll(packetTopicNames.values)
+}
+
+//fun KafkatorioPacket2.topicName(): String = data.topicName()
+
+//fun KafkatorioPacketData2.topicName(): String = "$DOMAIN.packet.${topicName}"
+
+
+val KafkatorioPacketData2.topicName: String
+  get() = when (this) {
+    is ConfigurationUpdate  -> packetTopicNames.getValue(ConfigurationUpdate::class)
+    is ConsoleChatUpdate    -> packetTopicNames.getValue(ConsoleChatUpdate::class)
+    is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class)
+    is PrototypesUpdate     -> packetTopicNames.getValue(PrototypesUpdate::class)
+    is SurfaceUpdate        -> packetTopicNames.getValue(SurfaceUpdate::class)
+    is EntityUpdate         -> packetTopicNames.getValue(EntityUpdate::class)
+    is MapChunkUpdate       -> packetTopicNames.getValue(MapChunkUpdate::class)
+    is PlayerUpdate         -> packetTopicNames.getValue(PlayerUpdate::class)
+  }
+
+
+private val packetTopicNames: Map<KClass<out KafkatorioPacketData2>, String> = mapOf(
+  ConfigurationUpdate::class to "configuration",
+  ConsoleChatUpdate::class to "console-chat",
+  ConsoleCommandUpdate::class to "console-command",
+  PrototypesUpdate::class to "prototypes",
+  SurfaceUpdate::class to "surface",
+  EntityUpdate::class to "entity",
+  MapChunkUpdate::class to "map-chunk",
+  PlayerUpdate::class to "player",
+).mapValues { (_, v) ->
+  "$DOMAIN.packet.$v"
+}
+
+val KClass<out KafkatorioPacketData2>.topicName: String
+  get() = packetTopicNames.getValue(this)
+
+//fun packetTopic(
+//  dataType: KafkatorioKeyedPacketData.KafkatorioKeyedPacketDataType,
+//): String = packetTopic(KafkatorioPacketType.KEYED, dataType.name.lowercase())
+//
+//private fun packetTopic(
+//  packetType: KafkatorioPacketType,
+//  dataType: String,
+//): String {
+//  return "$DOMAIN.packet.$packetType.$dataType"
+//}
+
+//sealed interface KafkatorioTopic {
+//
+//  val type: String
+//  val entity: String
+//  val description: String
+//
+//  companion object {
+//    private const val DOMAIN = "kafkatorio"
+//
+//    val KafkatorioTopic.topic
+//      get() = listOf(DOMAIN, type, entity, description).joinToString(".")
+//
+//  }
+//
+//  data class Event(
+//    override val entity: String,
+//    override val description: String,
+//  ) : KafkatorioTopic {
+//    override val type: String = "event"
+//  }
+//
+//
+//  data class State(
+//    override val entity: String,
+//    override val description: String,
+//  ) : KafkatorioTopic {
+//    override val type: String = "state"
+//  }
+//
+//}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes/kxsBinary.kt

@@ -1,11 +1,5 @@
 package dev.adamko.kafkatorio.processor.serdes
 
-import java.io.ByteArrayInputStream
-import java.io.ByteArrayOutputStream
-import java.io.DataInput
-import java.io.DataInputStream
-import java.io.DataOutput
-import java.io.DataOutputStream
 import kotlinx.serialization.BinaryFormat
 import kotlinx.serialization.DeserializationStrategy
 import kotlinx.serialization.SerializationStrategy
@@ -17,6 +11,9 @@ import kotlinx.serialization.encoding.CompositeDecoder
 import kotlinx.serialization.encoding.CompositeEncoder
 import kotlinx.serialization.modules.EmptySerializersModule
 import kotlinx.serialization.modules.SerializersModule
+import okio.Buffer
+import okio.BufferedSink
+import okio.BufferedSource
 
 
 val kxsBinary: KxsBinary = KxsBinary()
@@ -27,18 +24,18 @@ class KxsBinary(
 ) : BinaryFormat {
 
   override fun <T> encodeToByteArray(serializer: SerializationStrategy<T>, value: T): ByteArray {
-    val output = ByteArrayOutputStream()
+    val output = Buffer()
-    val encoder = KxsDataOutputEncoder(DataOutputStream(output), serializersModule)
+    val encoder = KxsDataOutputEncoder(output, serializersModule)
     encoder.encodeSerializableValue(serializer, value)
-    return output.toByteArray()
+    return output.readByteArray()
   }
 
   override fun <T> decodeFromByteArray(
     deserializer: DeserializationStrategy<T>,
     bytes: ByteArray
   ): T {
-    val input = ByteArrayInputStream(bytes)
+    val input = Buffer().write(bytes)
-    val decoder = KxsDataInputDecoder(DataInputStream(input), serializersModule)
+    val decoder = KxsDataInputDecoder(input, serializersModule)
     return decoder.decodeSerializableValue(deserializer)
   }
 }
@@ -46,28 +43,31 @@ class KxsBinary(
 private val byteArraySerializer = ByteArraySerializer()
 
 class KxsDataOutputEncoder(
-  private val output: DataOutput,
+  private val output: BufferedSink,
   override val serializersModule: SerializersModule = EmptySerializersModule,
 ) : AbstractEncoder() {
 
-  override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
+  private operator fun BufferedSink.invoke(action: BufferedSink.() -> Unit): Unit = action()
-  override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
+
-  override fun encodeChar(value: Char) = output.writeChar(value.code)
+  override fun encodeBoolean(value: Boolean) = output { writeByte(if (value) 1 else 0) }
-  override fun encodeDouble(value: Double) = output.writeDouble(value)
+  override fun encodeByte(value: Byte) = output { writeByte(value.toInt()) }
-  override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output.writeInt(index)
+  override fun encodeChar(value: Char) = output { writeUtf8CodePoint(value.code) }
-  override fun encodeFloat(value: Float) = output.writeFloat(value)
+  override fun encodeDouble(value: Double) = output { writeLong(value.toRawBits()) }
-  override fun encodeInt(value: Int) = output.writeInt(value)
+  override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output { writeInt(index) }
-  override fun encodeLong(value: Long) = output.writeLong(value)
+  override fun encodeFloat(value: Float) = output { writeInt(value.toRawBits()) }
-  override fun encodeShort(value: Short) = output.writeShort(value.toInt())
+  override fun encodeInt(value: Int) = output { writeInt(value) }
-  override fun encodeString(value: String) = output.writeUTF(value)
+  override fun encodeLong(value: Long) = output { writeLong(value) }
-//  override fun encodeString(value: String) = encodeByteArray(value.toByteArray(Charsets.UTF_8))
+  override fun encodeShort(value: Short) = output { writeShort(value.toInt()) }
+  override fun encodeString(value: String) = output {
+    encodeCompactSize(value.length)
+    writeUtf8(value)
+  }
 
   override fun beginCollection(
     descriptor: SerialDescriptor,
     collectionSize: Int
   ): CompositeEncoder {
-    encodeInt(collectionSize)
+    encodeCompactSize(collectionSize)
-//    encodeCompactSize(collectionSize)
     return this
   }
 
@@ -98,7 +98,7 @@ class KxsDataOutputEncoder(
 
 
 class KxsDataInputDecoder(
-  private val input: DataInput,
+  private val input: BufferedSource,
   override val serializersModule: SerializersModule = EmptySerializersModule,
   private var elementsCount: Int = 0,
 ) : AbstractDecoder() {
@@ -107,15 +107,17 @@ class KxsDataInputDecoder(
 
   override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0
   override fun decodeByte(): Byte = input.readByte()
-  override fun decodeChar(): Char = input.readChar()
+  override fun decodeChar(): Char = input.readUtf8CodePoint().toChar()
-  override fun decodeDouble(): Double = input.readDouble()
+  override fun decodeDouble(): Double = Double.fromBits(input.readLong())
   override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readInt()
-  override fun decodeFloat(): Float = input.readFloat()
+  override fun decodeFloat(): Float = Float.fromBits(input.readInt())
   override fun decodeInt(): Int = input.readInt()
   override fun decodeLong(): Long = input.readLong()
   override fun decodeShort(): Short = input.readShort()
-  override fun decodeString(): String = input.readUTF()
+  override fun decodeString(): String {
-//  override fun decodeString(): String = decodeByteArray().toString(Charsets.UTF_8)
+    val size = decodeCompactSize()
+    return input.readUtf8(size.toLong())
+  }
 
   override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
     if (elementIndex == elementsCount) return CompositeDecoder.DECODE_DONE
@@ -128,8 +130,7 @@ class KxsDataInputDecoder(
   override fun decodeSequentially(): Boolean = true
 
   override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
-    decodeInt().also { elementsCount = it }
+    decodeCompactSize().also { elementsCount = it }
-//    decodeCompactSize().also { elementsCount = it }
 
   override fun decodeNotNullMark(): Boolean = decodeBoolean()
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes/kxsJson.kt

@@ -1,7 +1,7 @@
 package dev.adamko.kafkatorio.processor.serdes
 
-import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
+import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
-import dev.adamko.kafkatorio.events.schema.jsonMapperKafkatorio
+import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
@@ -13,15 +13,15 @@ import org.http4k.format.asConfigurable
 import org.http4k.format.withStandardMappings
 
 
-object KafkatorioPacketSerde : Serde<KafkatorioPacket> {
+//object KafkatorioPacketSerde : Serde<KafkatorioPacket2> {
-  override fun serializer() = Serializer<KafkatorioPacket> { _, message ->
+//  override fun serializer() = Serializer<KafkatorioPacket> { _, message ->
-    jsonMapper.encodeToString<KafkatorioPacket>(message).encodeToByteArray()
+//    jsonMapper.encodeToString<KafkatorioPacket>(message).encodeToByteArray()
-  }
+//  }
-
+//
-  override fun deserializer() = Deserializer<KafkatorioPacket> { _, bytes ->
+//  override fun deserializer() = Deserializer<KafkatorioPacket> { _, bytes ->
-    jsonMapper.decodeFromString(bytes.decodeToString())
+//    jsonMapper.decodeFromString(bytes.decodeToString())
-  }
+//  }
-}
+//}
 
 val jsonMapper: Json = Json {
   prettyPrint = true

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt

@@ -1,13 +1,10 @@
 package dev.adamko.kafkatorio.processor.topology
 
-import dev.adamko.kafkatorio.events.schema.FactorioConfigurationUpdate
+
-import dev.adamko.kafkatorio.events.schema.FactorioEvent
+import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG
-import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
+import dev.adamko.kafkatorio.processor.admin.topicName
-import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
-import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
-import dev.adamko.kafkatorio.processor.KafkatorioTopology
-import dev.adamko.kafkatorio.processor.admin.KafkatorioKafkaAdmin
 import dev.adamko.kafkatorio.processor.serdes.jsonMapper
+import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
 import dev.adamko.kotka.extensions.consumedAs
 import dev.adamko.kotka.extensions.producedAs
 import dev.adamko.kotka.extensions.streams.map
@@ -22,25 +19,24 @@ import org.apache.kafka.streams.processor.RecordContext
 
 fun factorioServerPacketStream(
   builder: StreamsBuilder = StreamsBuilder(),
-): KStream<FactorioServerId, KafkatorioPacket> {
+): KStream<FactorioServerId, KafkatorioPacket2> {
 
   return builder.stream(
-    KafkatorioTopology.TOPIC_SRC_SERVER_LOG,
+    TOPIC_SRC_SERVER_LOG,
     consumedAs(
       "read-raw-packets-from-server",
       Serdes.String(),
       Serdes.String(),
     )
-  )
+  ).map("decode-packets") { serverId: String, value: String ->
-    .map("decode-packets") { serverId: String, value: String ->
+    val packet = jsonMapper.decodeFromString<KafkatorioPacket2>(value)
-      val packet = jsonMapper.decodeFromString<KafkatorioPacket>(value)
     FactorioServerId(serverId) to packet
   }
 }
 
 
 fun splitFactorioServerPacketStream(
-  factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>
+  factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket2>
 ) {
   factorioServerPacketStream
     .to(
@@ -49,18 +45,8 @@ fun splitFactorioServerPacketStream(
         jsonMapper.serde(),
         jsonMapper.serde(),
       )
-    ) { _: FactorioServerId, value: KafkatorioPacket, _: RecordContext ->
+    ) { _: FactorioServerId, value: KafkatorioPacket2, _: RecordContext ->
 //        println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}")
-
+      value.data.topicName
-      val base = KafkatorioKafkaAdmin.topicNameBase(value.packetType)
-
-      val type: String = when (value) {
-        is FactorioEvent               -> value.data.objectName.name
-        is FactorioConfigurationUpdate -> "FactorioConfigurationUpdate"
-        is FactorioPrototypes          -> "all"
-        is FactorioEventUpdatePacket   -> value.update.updateType.name
-      }
-
-      "$base.$type"
     }
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/flowMapTiles.kt

@@ -4,12 +4,8 @@ import com.sksamuel.scrimage.ImmutableImage
 import com.sksamuel.scrimage.ScaleMethod
 import com.sksamuel.scrimage.color.RGBColor
 import com.sksamuel.scrimage.nio.PngWriter
-import dev.adamko.kafkatorio.events.schema.ColourHex
+import dev.adamko.kafkatorio.schema.common.*
-import dev.adamko.kafkatorio.events.schema.MapChunkPosition
+import dev.adamko.kafkatorio.processor.admin.TOPIC_GROUPED_MAP_CHUNKS_STATE
-import dev.adamko.kafkatorio.events.schema.MapTilePosition
-import dev.adamko.kafkatorio.events.schema.SurfaceIndex
-import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
-import dev.adamko.kafkatorio.processor.KafkatorioTopology
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
 import dev.adamko.kotka.extensions.consumedAs
 import dev.adamko.kotka.extensions.streams.forEach
@@ -17,7 +13,9 @@ import dev.adamko.kotka.kxs.serde
 import java.awt.image.BufferedImage
 import java.io.File
 import kotlin.coroutines.CoroutineContext
+import kotlin.io.path.fileSize
 import kotlin.math.abs
+import kotlin.time.Duration.Companion.seconds
 import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -25,17 +23,18 @@ import kotlinx.coroutines.SupervisorJob
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.channels.BufferOverflow
+import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.MutableSharedFlow
 import kotlinx.coroutines.flow.asFlow
+import kotlinx.coroutines.flow.debounce
+import kotlinx.coroutines.flow.distinctUntilChangedBy
 import kotlinx.coroutines.flow.emitAll
 import kotlinx.coroutines.flow.flatMapConcat
 import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.runningFold
-import kotlinx.coroutines.job
 import kotlinx.coroutines.launch
-import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.supervisorScope
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.Topology
@@ -44,10 +43,6 @@ import org.apache.kafka.streams.kstream.KStream
 
 private const val WEB_MAP_TILE_SIZE_PX = 256
 
-private val cScope = CoroutineScope(
-  Dispatchers.Default + SupervisorJob() + CoroutineName("f-server-map-updates")
-)
-
 
 /** A bridge between Kafka and Kotlin */
 private val serverMapChunkHandler = ServerMapChunkHandler()
@@ -59,19 +54,17 @@ fun saveMapTiles(
 
   val groupedMapChunkTiles: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
     builder.stream(
-      KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS,
+      TOPIC_GROUPED_MAP_CHUNKS_STATE,
       consumedAs("consume.grouped-map-chunks", kxsBinary.serde(), kxsBinary.serde())
     )
 
   groupedMapChunkTiles
     .forEach("save-chunked-tiles") { _, value ->
       if (value != null) {
-        runBlocking(Dispatchers.Default) {
         println("emitting chunkedTiles ${value.chunkId} tiles: ${value.map.size}")
         serverMapChunkHandler.emit(value)
       }
     }
-    }
 
   return builder.build()
 }
@@ -107,9 +100,8 @@ private fun ColourHex.toRgbColor(): RGBColor {
 
 private class ServerMapChunkHandler : CoroutineScope {
 
-  override val coroutineContext: CoroutineContext = Dispatchers.Default +
+  override val coroutineContext: CoroutineContext =
-      SupervisorJob(cScope.coroutineContext.job) +
+    Dispatchers.Default + SupervisorJob() + CoroutineName("ServerMapChunkHandler")
-      CoroutineName("ServerMapChunkHandler")
 
 
   /** The inbox for this handler. */
@@ -119,11 +111,9 @@ private class ServerMapChunkHandler : CoroutineScope {
     onBufferOverflow = BufferOverflow.SUSPEND,
   )
 
-  fun emit(mapChunkTiles: ServerMapChunkTiles<ColourHex>) {
+  fun emit(mapChunkTiles: ServerMapChunkTiles<ColourHex>) = launch {
-    launch {
     allChunksFlow.emit(mapChunkTiles)
   }
-  }
 
   /** This flow saves images one-by-one */
   private val saveImagesFlow = MutableSharedFlow<Cmd.SaveImage>(
@@ -135,17 +125,18 @@ private class ServerMapChunkHandler : CoroutineScope {
   init {
 
     allChunksFlow
-      .runningFold(mapOf<ServerMapChunkId, MutableSharedFlow<Cmd.ChunkSubdivide>>()) { acc, src ->
+      .runningFold(mutableMapOf<ServerMapChunkId, MutableSharedFlow<Cmd.ChunkSubdivide>>()) { acc, src ->
 
-        val flow = acc.getOrElse(src.chunkId) {
+        val flow = acc.getOrPut(src.chunkId) {
           println("creating new subdivide-flow for chunkId ${src.chunkId}")
           createSubdivisionFlow()
         }
 
         flow.emit(Cmd.ChunkSubdivide(src.chunkId, src.map))
 
-        acc + (src.chunkId to flow)
+        acc
       }
+      .distinctUntilChangedBy { it.size }
       .onEach {
         println("allChunksFlow has ${it.size} subdivide-flows: ${it.keys.joinToString()}")
       }
@@ -160,9 +151,15 @@ private class ServerMapChunkHandler : CoroutineScope {
           println("created new map tile parentFile directory ${chunkImageFile.absolutePath}")
         }
 
+        val sizeBefore = chunkImageFile.takeIf { it.exists() }?.toPath()?.fileSize()
+
         val savedTile = img.output(PngWriter.NoCompression, chunkImageFile)
-        if (filename.value.contains("z-1")) {
+
-          println("savedTile: $savedTile")
+        val sizeAfter = savedTile.takeIf { it.exists() }?.toPath()?.fileSize()
+        if (sizeBefore != sizeAfter) {
+//          println("savedTile $sizeBefore/$sizeAfter: $savedTile")
+        } else {
+          println("savedTile NO-SIZE-CHANGE $sizeBefore/$sizeAfter: $savedTile")
         }
       }
       .launchIn(this)
@@ -183,17 +180,20 @@ private class ServerMapChunkHandler : CoroutineScope {
       onBufferOverflow = BufferOverflow.SUSPEND,
     )
 
-    val saveImgCommands = subdivisionFlow
+    val saveImgCommands: Flow<Cmd.SaveImage> =
+      subdivisionFlow
+        .debounce(30.seconds)
         .flatMapConcat { subdivideCmd ->
           ChunkSize
-          .values
+            .entries
             .flatMap { chunkSize ->
-            subdivideCmd.tiles
+              subdivideCmd
+                .tiles
                 .entries
                 .groupBy(
                   { (tile, _) -> tile.toMapChunkPosition(chunkSize.tilesPerChunk) }
-              ) {
+                ) { (tilePosition, colour) ->
-                it.key to it.value
+                  tilePosition to colour
                 }
                 .map { (chunkPos, tiles) ->
                   async {

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt

@@ -1,52 +1,31 @@
 package dev.adamko.kafkatorio.processor.topology
 
-import dev.adamko.kafkatorio.events.schema.ColourHex
+import dev.adamko.kafkatorio.processor.admin.TOPIC_GROUPED_MAP_CHUNKS_STATE
-import dev.adamko.kafkatorio.events.schema.FactorioEventUpdate
-import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
-import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
-import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
-import dev.adamko.kafkatorio.events.schema.MapChunkPosition
-import dev.adamko.kafkatorio.events.schema.MapChunkUpdate
-import dev.adamko.kafkatorio.events.schema.MapTile
-import dev.adamko.kafkatorio.events.schema.MapTileDictionary.Companion.isNullOrEmpty
-import dev.adamko.kafkatorio.events.schema.MapTilePosition
-import dev.adamko.kafkatorio.events.schema.MapTiles
-import dev.adamko.kafkatorio.events.schema.SurfaceIndex
-import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
-import dev.adamko.kafkatorio.processor.KafkatorioTopology
-import dev.adamko.kafkatorio.processor.serdes.jsonMapper
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
-import dev.adamko.kotka.extensions.consumedAs
+import dev.adamko.kafkatorio.schema.common.ColourHex
+import dev.adamko.kafkatorio.schema.common.MapChunkPosition
+import dev.adamko.kafkatorio.schema.common.MapTile
+import dev.adamko.kafkatorio.schema.common.MapTileDictionary.Companion.isNullOrEmpty
+import dev.adamko.kafkatorio.schema.common.MapTilePosition
+import dev.adamko.kafkatorio.schema.common.MapTiles
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.common.toMapChunkPosition
+import dev.adamko.kafkatorio.schema2.MapChunkUpdate
+import dev.adamko.kafkatorio.schema2.PrototypesUpdate
 import dev.adamko.kotka.extensions.groupedAs
 import dev.adamko.kotka.extensions.materializedWith
 import dev.adamko.kotka.extensions.producedAs
-import dev.adamko.kotka.extensions.repartitionedAs
 import dev.adamko.kotka.extensions.streams.filter
 import dev.adamko.kotka.extensions.streams.flatMap
 import dev.adamko.kotka.extensions.streams.mapValues
 import dev.adamko.kotka.extensions.streams.peek
 import dev.adamko.kotka.extensions.streams.reduce
-import dev.adamko.kotka.extensions.streams.toTable
 import dev.adamko.kotka.extensions.tableJoined
-import dev.adamko.kotka.extensions.tables.filter
 import dev.adamko.kotka.extensions.tables.join
-import dev.adamko.kotka.extensions.tables.mapValues
 import dev.adamko.kotka.extensions.tables.toStream
 import dev.adamko.kotka.kxs.serde
-import kotlin.collections.List
-import kotlin.collections.Map
-import kotlin.collections.associate
 import kotlin.collections.component1
 import kotlin.collections.component2
-import kotlin.collections.getOrElse
-import kotlin.collections.groupBy
-import kotlin.collections.isNotEmpty
-import kotlin.collections.isNullOrEmpty
-import kotlin.collections.joinToString
-import kotlin.collections.map
-import kotlin.collections.mapValues
-import kotlin.collections.mutableSetOf
-import kotlin.collections.plus
 import kotlinx.serialization.Serializable
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.Topology
@@ -72,33 +51,39 @@ data class ServerMapChunkTiles<Data>(
 }
 
 
-fun groupMapChunks(
+fun groupMapChunks(builder: StreamsBuilder): Topology {
-  builder: StreamsBuilder,
+
-  prototypesTopicName: String = "kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
+  val mapChunksStream: KStream<FactorioServerId, MapChunkUpdate> = builder.streamPacketData()
-  mapChunkUpdatesTopicName: String = "kafkatorio.${KafkatorioPacket.PacketType.UPDATE}.${FactorioEventUpdate.FactorioEventUpdateType.MAP_CHUNK}",
+  val protosStream: KStream<FactorioServerId, PrototypesUpdate> = builder.streamPacketData()
-): Topology {
+
+//  val protosStream: KStream<FactorioServerId, KafkatorioPacket2> =
+//    builder.stream<FactorioServerId?, KafkatorioPacket2?>(
+//      prototypesTopicName,
+//      consumedAs("consume.factorio-protos.all", jsonMapper.serde(), jsonMapper.serde())
+//    ).filter { key: FactorioServerId?, value: KafkatorioPacket2? ->
+//      key != null
+//          && value != null
+//          && value.data ==
+//    }
+//      .map
+
 
-  val protosStream: KStream<FactorioServerId, FactorioPrototypes> =
-    builder.stream(
-      prototypesTopicName,
-      consumedAs("consume.factorio-protos.all", jsonMapper.serde(), jsonMapper.serde())
-    )
   val tileProtoColourDict: KTable<FactorioServerId, TileColourDict> =
     tileProtoColourDictionary(protosStream)
 
-  val mapChunksUpdatesStream: KStream<FactorioServerId, FactorioEventUpdatePacket> =
+//  val mapChunksUpdatesStream: KStream<FactorioServerId, FactorioEventUpdatePacket> =
-    builder.stream(
+//    builder.stream(
-      mapChunkUpdatesTopicName,
+//      mapChunkUpdatesTopicName,
-      consumedAs(
+//      consumedAs(
-        "consume.map-chunk-updates",
+//        "consume.map-chunk-updates",
-        jsonMapper.serde<FactorioServerId>(),
+//        jsonMapper.serde<FactorioServerId>(),
-        jsonMapper.serde<FactorioEventUpdatePacket>(),
+//        jsonMapper.serde<FactorioEventUpdatePacket>(),
-      )
+//      )
-    )
+//    )
 
   val groupedMapChunkTiles: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
     groupTilesIntoChunksWithColours(
-      mapChunksUpdatesStream,
+      mapChunksStream,
       tileProtoColourDict,
     )
 
@@ -108,13 +93,13 @@ fun groupMapChunks(
       !v?.map.isNullOrEmpty()
     }
     .mapValues("stream-grouped-map-tiles.map-not-null") { _, v ->
-      v as ServerMapChunkTiles<ColourHex>
+      requireNotNull(v)
     }
     .peek("stream-grouped-map-tiles.peek") { _, v ->
       println("Grouping map tiles result: ${v.chunkId} / ${v.map.size}")
     }
     .to(
-      KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS,
+      TOPIC_GROUPED_MAP_CHUNKS_STATE,
       producedAs(
         "produce.grouped-map-chunks",
         kxsBinary.serde<ServerMapChunkId>(),
@@ -127,37 +112,24 @@ fun groupMapChunks(
 
 
 private fun groupTilesIntoChunksWithColours(
-  mapChunkUpdatePacketsStream: KStream<FactorioServerId, FactorioEventUpdatePacket>,
+  mapChunksStream: KStream<FactorioServerId, MapChunkUpdate>,
   tileProtoColourDict: KTable<FactorioServerId, TileColourDict>,
 ): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
 
-  val mapChunkUpdatesStream: KStream<FactorioServerId, MapTiles> =
-    mapChunkUpdatePacketsStream
-      .filter { _, packet: FactorioEventUpdatePacket? ->
-        when (val data = packet?.update) {
-          is MapChunkUpdate -> !data.tileDictionary.isNullOrEmpty()
-          else              -> false
-        }
-      }
-      .mapValues("map-chunk-update-packets.convert-to-map-tiles") { _, packet: FactorioEventUpdatePacket ->
 
-        val mapChunkUpdate: MapChunkUpdate = requireNotNull(packet.update as? MapChunkUpdate) {
+  val mapTilesStream: KStream<FactorioServerId, MapTiles> =
-          "invalid UpdatePacket type $packet"
+    mapChunksStream.filter { _, value ->
-        }
+      !value.tileDictionary.isNullOrEmpty()
-
+    }.mapValues("map-chunk-update-packets.convert-to-map-tiles") { _, packet: MapChunkUpdate ->
-        val tiles: List<MapTile> = requireNotNull(mapChunkUpdate.tileDictionary?.toMapTileList()) {
+      val mapTiles = packet.tileDictionary?.toMapTileList() ?: emptyList()
-          "invalid MapChunkUpdate packet, tiles must not be null"
+      MapTiles(packet.key.surfaceIndex, mapTiles)
-        }
+    }.filter { _, value ->
-        require(tiles.isNotEmpty()) {
+      value.tiles.isNotEmpty()
-          "invalid MapChunkUpdate packet, tiles must not be empty"
-        }
-
-        MapTiles(mapChunkUpdate.surfaceIndex, tiles)
     }
 
 
   val chunkedTilesTable: KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>> =
-    mapChunkUpdatesStream
+    mapTilesStream
       // group tiles by server & surface & chunk
       .flatMap("server-map-data.tiles.flatMapByChunk") { serverId: FactorioServerId, mapTiles: MapTiles ->
 
@@ -208,65 +180,65 @@ private fun groupTilesIntoChunksWithColours(
           kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
         )
       ) { chunkTiles, other -> chunkTiles + other }
-      .toStream("server-map-data.tiles.repartition.to-stream")
+//      .toStream("server-map-data.tiles.repartition.to-stream")
-      .repartition(
+//      .repartition(
-        repartitionedAs(
+//        repartitionedAs(
-          "server-map-data.tiles.repartition.config",
+//          "server-map-data.tiles.repartition.config",
-          kxsBinary.serde<ServerMapChunkId>(),
+//          kxsBinary.serde<ServerMapChunkId>(),
-          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
+//          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
-          3,
+//          3,
-        )
+//        )
-      ).toTable(
+//      ).toTable(
-        "server-map-data.tiles.repartition.to-table",
+//        "server-map-data.tiles.repartition.to-table",
-        materializedWith(
+//        materializedWith(
-          kxsBinary.serde<ServerMapChunkId>(),
+//          kxsBinary.serde<ServerMapChunkId>(),
-          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
+//          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
-        )
+//        )
-      )
+//      )
-      .filter("server-map-data.tiles.repartition.filter-not-null") { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
+//      .filter("server-map-data.tiles.repartition.filter-not-null") { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
-        v != null
+//        v != null
-      }
+//      }
-      .mapValues(
+//      .mapValues(
-        "server-map-data.tiles.repartition.map-not-null",
+//        "server-map-data.tiles.repartition.map-not-null",
-        materializedWith(
+//        materializedWith(
-          kxsBinary.serde<ServerMapChunkId>(),
+//          kxsBinary.serde<ServerMapChunkId>(),
-          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
+//          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
-        )
+//        )
-      ) { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
+//      ) { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
-        v as ServerMapChunkTiles<TileProtoHashCode>
+//        v as ServerMapChunkTiles<TileProtoHashCode>
-      }
+//      }
 
 
   // repartition to try and fix KTable-KTable joining skipping records
   val tileProtoColourDictRepartitioned =
     tileProtoColourDict
-      .toStream("tileProtoColourDict.repartition.to-stream")
+//      .toStream("tileProtoColourDict.repartition.to-stream")
-      .repartition(
+//      .repartition(
-        repartitionedAs(
+//        repartitionedAs(
-          "tileProtoColourDict.repartition.config",
+//          "tileProtoColourDict.repartition.config",
-          kxsBinary.serde<FactorioServerId>(),
+//          kxsBinary.serde<FactorioServerId>(),
-          kxsBinary.serde<TileColourDict?>(),
+//          kxsBinary.serde<TileColourDict?>(),
-          3,
+//          3,
-        )
+//        )
-      ).toTable(
+//      ).toTable(
-        "tileProtoColourDict.repartition.to-table",
+//        "tileProtoColourDict.repartition.to-table",
-        materializedWith(
+//        materializedWith(
-          kxsBinary.serde<FactorioServerId>(),
+//          kxsBinary.serde<FactorioServerId>(),
-          kxsBinary.serde<TileColourDict?>(),
+//          kxsBinary.serde<TileColourDict?>(),
-        )
+//        )
-      )
+//      )
-      .filter("tileProtoColourDict.repartition.filter-not-null") { _: FactorioServerId, v: TileColourDict? ->
+//      .filter("tileProtoColourDict.repartition.filter-not-null") { _: FactorioServerId, v: TileColourDict? ->
-        v != null
+//        v != null
-      }
+//      }
-      .mapValues(
+//      .mapValues(
-        "tileProtoColourDict.repartition.map-not-null",
+//        "tileProtoColourDict.repartition.map-not-null",
-        materializedWith(
+//        materializedWith(
-          kxsBinary.serde<FactorioServerId>(),
+//          kxsBinary.serde<FactorioServerId>(),
-          kxsBinary.serde<TileColourDict>(),
+//          kxsBinary.serde<TileColourDict>(),
-        )
+//        )
-      ) { _: FactorioServerId, v: TileColourDict? ->
+//      ) { _: FactorioServerId, v: TileColourDict? ->
-        v as TileColourDict
+//        v as TileColourDict
-      }
+//      }
 
 
   val colourisedChunkTable: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/model.kt

@@ -36,11 +36,11 @@ enum class ChunkSize(
 
   companion object {
     // cache values for better performance. KT-48872
-    val values: List<ChunkSize> = values().asList()
+    val entries: Set<ChunkSize> = values().toSet()
 
-    val MAX: ChunkSize = values.maxByOrNull { it.tilesPerChunk }!!
+    val MAX: ChunkSize = entries.maxByOrNull { it.tilesPerChunk }!!
-    val MIN: ChunkSize = values.minByOrNull { it.tilesPerChunk }!!
+    val MIN: ChunkSize = entries.minByOrNull { it.tilesPerChunk }!!
-    val STANDARD: ChunkSize = values.first { it.zoomLevel == 0 }
+    val STANDARD: ChunkSize = entries.first { it.zoomLevel == 0 }
 
   }
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/packetDataStreams.kt

@@ -0,0 +1,27 @@
+package dev.adamko.kafkatorio.processor.topology
+
+import dev.adamko.kafkatorio.processor.admin.topicName
+import dev.adamko.kafkatorio.processor.serdes.jsonMapper
+import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
+import dev.adamko.kafkatorio.schema2.KafkatorioPacketData2
+import dev.adamko.kotka.extensions.consumedAs
+import dev.adamko.kotka.extensions.streams.mapValues
+import dev.adamko.kotka.kxs.serde
+import org.apache.kafka.streams.StreamsBuilder
+import org.apache.kafka.streams.kstream.KStream
+
+
+inline fun <reified T : KafkatorioPacketData2> StreamsBuilder.streamPacketData(): KStream<FactorioServerId, T> {
+
+  val simpleName = T::class.simpleName!!
+  val topicName = T::class.topicName
+
+  return stream<FactorioServerId, KafkatorioPacket2>(
+    topicName,
+    consumedAs("consume.${simpleName}", jsonMapper.serde(), jsonMapper.serde())
+  ).filter { key: FactorioServerId?, value: KafkatorioPacket2? ->
+    key != null && value?.data is T
+  }.mapValues("consume.map-values.${simpleName}") { _: FactorioServerId, packet: KafkatorioPacket2 ->
+    packet.data as T
+  }
+}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/playerUpdates.kt

@@ -1,9 +1,10 @@
 package dev.adamko.kafkatorio.processor.topology
 
-import dev.adamko.kafkatorio.events.schema.FactorioEventUpdate
-import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
 import dev.adamko.kafkatorio.processor.WebsocketServer
+import dev.adamko.kafkatorio.processor.admin.topicName
 import dev.adamko.kafkatorio.processor.serdes.jsonMapper
+import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
+import dev.adamko.kafkatorio.schema2.PlayerUpdate
 import dev.adamko.kotka.extensions.consumedAs
 import dev.adamko.kotka.kxs.serde
 import kotlinx.serialization.encodeToString
@@ -14,11 +15,10 @@ fun playerUpdatesToWsServer(
   websocketServer: WebsocketServer,
   builder: StreamsBuilder = StreamsBuilder(),
 ) {
-  builder.stream<FactorioServerId, KafkatorioPacket>(
+  builder.stream<FactorioServerId, KafkatorioPacket2>(
-    "kafkatorio.${KafkatorioPacket.PacketType.UPDATE}.${FactorioEventUpdate.FactorioEventUpdateType.PLAYER}",
+    PlayerUpdate::class.topicName,
     consumedAs("EventUpdate.Player-updates-for-ws-server", jsonMapper.serde(), jsonMapper.serde()),
-  )
+  ).foreach { _, value ->
-    .foreach { _, value ->
 //      println("sending ${value.packetType} packet to websocket")
     websocketServer.sendMessage(jsonMapper.encodeToString(value))
   }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt

@@ -1,15 +1,13 @@
 package dev.adamko.kafkatorio.processor.topology
 
-import dev.adamko.kafkatorio.events.schema.ColourHex
-import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
-import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
-import dev.adamko.kafkatorio.events.schema.MapTile
-import dev.adamko.kafkatorio.events.schema.MapTilePrototype
-import dev.adamko.kafkatorio.events.schema.converters.toHex
-import dev.adamko.kafkatorio.processor.serdes.jsonMapper
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
+import dev.adamko.kafkatorio.schema.common.ColourHex
+import dev.adamko.kafkatorio.schema.common.MapTile
+import dev.adamko.kafkatorio.schema.common.toHex
+import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
+import dev.adamko.kafkatorio.schema.prototypes.MapTilePrototype
+import dev.adamko.kafkatorio.schema2.PrototypesUpdate
 import dev.adamko.kotka.extensions.materializedAs
-import dev.adamko.kotka.extensions.materializedWith
 import dev.adamko.kotka.extensions.streams.filter
 import dev.adamko.kotka.extensions.streams.mapValues
 import dev.adamko.kotka.extensions.streams.toTable
@@ -25,6 +23,7 @@ import org.apache.kafka.streams.kstream.KTable
 value class TileProtoHashCode private constructor(val code: Int) {
   constructor(prototype: MapTilePrototype) : this(prototype.name.hashCode())
   constructor(tile: MapTile) : this(tile.proto.hashCode())
+  constructor(tile: FactorioPrototype2.MapTile) : this(tile.name.hashCode())
 }
 
 
@@ -35,26 +34,21 @@ value class TileColourDict(val map: Map<TileProtoHashCode, ColourHex>)
 
 /** Get the latest map-tile colours per server. */
 fun tileProtoColourDictionary(
-  factorioServerPacketStream: KStream<FactorioServerId, FactorioPrototypes>
+  factorioServerPacketStream: KStream<FactorioServerId, PrototypesUpdate>
 ): KTable<FactorioServerId, TileColourDict> {
 
-  // get all the prototypes
+  return factorioServerPacketStream.mapValues("server-map-data.tile-prototypes.map-values") { _, protoPacket: PrototypesUpdate ->
-  return factorioServerPacketStream
+    val map = protoPacket
-    .mapValues("server-map-data.tile-prototypes.mapValues") { _, packet: KafkatorioPacket ->
-      val map = when (packet) {
-        is FactorioPrototypes -> packet
       .prototypes
-          .filterIsInstance<MapTilePrototype>()
+      .filterIsInstance<FactorioPrototype2.MapTile>()
       .associate { TileProtoHashCode(it) to it.mapColour.toHex() }
-        else                  -> mapOf()
+
-      }
     TileColourDict(map)
   }.filter("server-map-data.tile-prototypes.filterMapTileProtos") { _, dict: TileColourDict ->
     dict.map.isNotEmpty()
   }.peek { serverId, dict: TileColourDict ->
     println("server $serverId has TileColourDict[${dict.map.size}]: ${dict.map.entries.joinToString()}")
-    }
+  }.toTable(
-    .toTable(
     "server-map-data.tile-prototypes",
     materializedAs(
       "server-map-data.tile-prototypes.store",
@@ -62,5 +56,31 @@ fun tileProtoColourDictionary(
       kxsBinary.serde(),
     )
   )
-
 }
+
+// get all the prototypes
+//  return factorioServerPacketStream
+//    .mapValues("server-map-data.tile-prototypes.mapValues") { _, packet: KafkatorioPacket ->
+//      val map = when (packet) {
+//        is FactorioPrototype -> packet
+//          .prototypes
+//          .filterIsInstance<MapTilePrototype>()
+//          .associate { TileProtoHashCode(it) to it.mapColour.toHex() }
+//        else                 -> mapOf()
+//      }
+//      TileColourDict(map)
+//    }.filter("server-map-data.tile-prototypes.filterMapTileProtos") { _, dict: TileColourDict ->
+//      dict.map.isNotEmpty()
+//    }.peek { serverId, dict: TileColourDict ->
+//      println("server $serverId has TileColourDict[${dict.map.size}]: ${dict.map.entries.joinToString()}")
+//    }
+//    .toTable(
+//      "server-map-data.tile-prototypes",
+//      materializedAs(
+//        "server-map-data.tile-prototypes.store",
+//        kxsBinary.serde(),
+//        kxsBinary.serde(),
+//      )
+//    )
+//
+//}

modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/KxsBinaryTests.kt

@@ -1,24 +1,59 @@
 package dev.adamko.kafkatorio.processor.misc
 
-import dev.adamko.kafkatorio.events.schema.MapChunkPosition
-import dev.adamko.kafkatorio.events.schema.SurfaceIndex
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
 import dev.adamko.kafkatorio.processor.topology.ChunkSize
 import dev.adamko.kafkatorio.processor.topology.FactorioServerId
 import dev.adamko.kafkatorio.processor.topology.ServerMapChunkId
+import dev.adamko.kafkatorio.processor.topology.ServerMapChunkTiles
+import dev.adamko.kafkatorio.schema.common.ColourHex
+import dev.adamko.kafkatorio.schema.common.MapChunkPosition
+import dev.adamko.kafkatorio.schema.common.MapTilePosition
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
 import io.kotest.assertions.withClue
 import io.kotest.core.spec.style.FunSpec
+import io.kotest.matchers.collections.shouldContainExactly
+import io.kotest.matchers.comparables.shouldBeEqualComparingTo
 import io.kotest.matchers.equality.shouldBeEqualToComparingFields
+import io.kotest.matchers.ints.shouldBeExactly
+import io.kotest.matchers.longs.shouldBeExactly
 import io.kotest.matchers.shouldBe
+import io.kotest.property.Arb
+import io.kotest.property.arbitrary.boolean
+import io.kotest.property.arbitrary.booleanArray
+import io.kotest.property.arbitrary.byte
+import io.kotest.property.arbitrary.byteArray
+import io.kotest.property.arbitrary.char
+import io.kotest.property.arbitrary.charArray
+import io.kotest.property.arbitrary.double
+import io.kotest.property.arbitrary.doubleArray
+import io.kotest.property.arbitrary.float
+import io.kotest.property.arbitrary.floatArray
+import io.kotest.property.arbitrary.int
+import io.kotest.property.arbitrary.intArray
+import io.kotest.property.arbitrary.long
+import io.kotest.property.arbitrary.longArray
+import io.kotest.property.arbitrary.positiveInt
+import io.kotest.property.arbitrary.short
+import io.kotest.property.arbitrary.shortArray
+import io.kotest.property.arbitrary.string
+import io.kotest.property.arbitrary.uByte
+import io.kotest.property.arbitrary.uInt
+import io.kotest.property.arbitrary.uLong
+import io.kotest.property.arbitrary.uShort
+import io.kotest.property.checkAll
+import kotlinx.serialization.builtins.serializer
 import kotlinx.serialization.decodeFromByteArray
+import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.encodeToByteArray
+import kotlinx.serialization.encodeToString
 
 
 class KxsBinaryTests : FunSpec({
 
   test("byte array") {
     val ba = byteArrayOf(
-      0, 4, 116, 101, 115, 116, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
+      4, 116, 101, 115, 116, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
     )
 
     val result = kxsBinary.decodeFromByteArray(ServerMapChunkId.serializer(), ba)
@@ -47,7 +82,7 @@ class KxsBinaryTests : FunSpec({
 
     withClue(ba.toString(Charsets.UTF_8)) {
       ba shouldBe byteArrayOf(
-        0, 4, 116, 101, 115, 116, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
+        4, 116, 101, 115, 116, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
       )
     }
   }
@@ -55,7 +90,10 @@ class KxsBinaryTests : FunSpec({
   context("inline value class") {
     val uuid = "fa858d73-6205-44cf-aeeb-6bbf3148c4a7"
     //@formatter:off
-    val expectedBytes = byteArrayOf(0, 36, 102, 97, 56, 53, 56, 100, 55, 51, 45, 54, 50, 48, 53, 45, 52, 52, 99, 102, 45, 97, 101, 101, 98, 45, 54, 98, 98, 102, 51, 49, 52, 56, 99, 52, 97, 55)
+    val expectedBytes = byteArrayOf(
+       36, 102,  97,  56, 53, 56, 100, 55, 51,  45, 54, 50, 48, 53, 45, 52, 52, 99,
+      102, 45,  97, 101, 101, 98, 45,  54, 98, 98, 102, 51, 49, 52, 56, 99, 52, 97, 55,
+    )
     //@formatter:on
 
     test("encode") {
@@ -73,4 +111,208 @@ class KxsBinaryTests : FunSpec({
     }
   }
 
+  context("ServerMapChunkTiles") {
+    val initial = ServerMapChunkTiles(
+      ServerMapChunkId(
+        FactorioServerId("server-id"),
+        MapChunkPosition(33, 44),
+        SurfaceIndex(4u),
+        ChunkSize.CHUNK_032,
+      ),
+      mapOf(
+        MapTilePosition(1, 2) to ColourHex(1u, 2u, 3u, 4u),
+      )
+    )
+
+    test("!json round trip") {
+      val jsonEncoded = jsonMapperKafkatorio.encodeToString(initial)
+      val jsonDecoded =
+        jsonMapperKafkatorio.decodeFromString<ServerMapChunkTiles<ColourHex>>(jsonEncoded)
+
+      jsonDecoded shouldBeEqualToComparingFields initial
+    }
+
+    test("binary round trip") {
+      val binaryEncoded = kxsBinary.encodeToByteArray(initial)
+      val binaryDecoded = kxsBinary.decodeFromByteArray<ServerMapChunkTiles<ColourHex>>(binaryEncoded)
+
+      binaryDecoded shouldBeEqualToComparingFields initial
+    }
+  }
+
+  context("primitives round trip") {
+    test("double") {
+      checkAll(Arb.double()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Double = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("float") {
+      checkAll(Arb.float()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Float = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("int") {
+      checkAll(Arb.int()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Int = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeExactly input
+      }
+    }
+
+    test("long") {
+      checkAll(Arb.long()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Long = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeExactly input
+      }
+    }
+
+    test("short") {
+      checkAll(Arb.short()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Short = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("byte") {
+      checkAll(Arb.byte()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Byte = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("boolean") {
+      checkAll(Arb.boolean()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Boolean = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("char") {
+      checkAll(Arb.char()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: Char = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("string") {
+      checkAll(Arb.string()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: String = kxsBinary.decodeFromByteArray(encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+  }
+
+
+  @OptIn(ExperimentalUnsignedTypes::class)
+  context("unsigned numbers round trip") {
+    test("uint") {
+      checkAll(Arb.uInt()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(UInt.serializer(), input)
+        val decoded: UInt = kxsBinary.decodeFromByteArray(UInt.serializer(), encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("ulong") {
+      checkAll(Arb.uLong()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(ULong.serializer(), input)
+        val decoded: ULong = kxsBinary.decodeFromByteArray(ULong.serializer(), encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("ushort") {
+      checkAll(Arb.uShort()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(UShort.serializer(), input)
+        val decoded: UShort = kxsBinary.decodeFromByteArray(UShort.serializer(), encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+
+    test("ubyte") {
+      checkAll(Arb.uByte()) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(UByte.serializer(), input)
+        val decoded: UByte = kxsBinary.decodeFromByteArray(UByte.serializer(), encoded)
+        decoded shouldBeEqualComparingTo input
+      }
+    }
+  }
+
+  context("primitive arrays round trip") {
+    test("double array") {
+      checkAll(Arb.doubleArray(Arb.positiveInt(50), Arb.double())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: DoubleArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+
+    test("float array") {
+      checkAll(Arb.floatArray(Arb.positiveInt(50), Arb.float())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: FloatArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+
+    test("int array") {
+      checkAll(Arb.intArray(Arb.positiveInt(50), Arb.int())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: IntArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+
+    test("long array") {
+      checkAll(Arb.longArray(Arb.positiveInt(50), Arb.long())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: LongArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+
+    test("short array") {
+      checkAll(Arb.shortArray(Arb.positiveInt(50), Arb.short())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: ShortArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+
+    test("byte array") {
+      checkAll(Arb.byteArray(Arb.positiveInt(50), Arb.byte())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: ByteArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+
+    test("boolean array") {
+      checkAll(Arb.booleanArray(Arb.positiveInt(50), Arb.boolean())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: BooleanArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+
+    test("char array") {
+      checkAll(Arb.charArray(Arb.positiveInt(50), Arb.char())) { input ->
+        val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
+        val decoded: CharArray = kxsBinary.decodeFromByteArray(encoded)
+        decoded.asList() shouldContainExactly input.asList()
+      }
+    }
+  }
 })

modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/serdeSize.kt

@@ -1,14 +1,11 @@
 package dev.adamko.kafkatorio.processor.misc
 
-import dev.adamko.kafkatorio.events.schema.ColourHex
-import dev.adamko.kafkatorio.events.schema.MapChunkPosition
-import dev.adamko.kafkatorio.events.schema.MapTilePosition
-import dev.adamko.kafkatorio.events.schema.SurfaceIndex
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
 import dev.adamko.kafkatorio.processor.topology.ChunkSize
 import dev.adamko.kafkatorio.processor.topology.FactorioServerId
 import dev.adamko.kafkatorio.processor.topology.ServerMapChunkId
 import dev.adamko.kafkatorio.processor.topology.ServerMapChunkTiles
+import dev.adamko.kafkatorio.schema.common.*
 import java.util.UUID
 import kotlinx.serialization.encodeToByteArray
 import org.junit.jupiter.api.Test

modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/topology/GroupMapChunksTest.kt

@@ -1,21 +1,15 @@
 package dev.adamko.kafkatorio.processor.topology
 
-import dev.adamko.kafkatorio.events.schema.Colour
-import dev.adamko.kafkatorio.events.schema.ColourHex
-import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
-import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
-import dev.adamko.kafkatorio.events.schema.MapChunkPosition
-import dev.adamko.kafkatorio.events.schema.MapChunkUpdate
-import dev.adamko.kafkatorio.events.schema.MapTileDictionary
-import dev.adamko.kafkatorio.events.schema.MapTilePosition
-import dev.adamko.kafkatorio.events.schema.MapTilePrototype
-import dev.adamko.kafkatorio.events.schema.PrototypeKey
-import dev.adamko.kafkatorio.events.schema.PrototypeName
-import dev.adamko.kafkatorio.events.schema.SurfaceIndex
-import dev.adamko.kafkatorio.events.schema.Tick
 import dev.adamko.kafkatorio.processor.KafkatorioTopology
+import dev.adamko.kafkatorio.processor.admin.TOPIC_GROUPED_MAP_CHUNKS_STATE
 import dev.adamko.kafkatorio.processor.serdes.jsonMapper
 import dev.adamko.kafkatorio.processor.serdes.kxsBinary
+import dev.adamko.kafkatorio.schema.common.*
+import dev.adamko.kafkatorio.schema.common.MapTileDictionary.PrototypeKey
+import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
+import dev.adamko.kafkatorio.schema2.MapChunkUpdate
+import dev.adamko.kafkatorio.schema2.MapChunkUpdateKey
+import dev.adamko.kafkatorio.schema2.PrototypesUpdate
 import dev.adamko.kotka.kxs.serde
 import io.kotest.assertions.withClue
 import io.kotest.core.spec.style.FunSpec
@@ -32,34 +26,29 @@ import org.apache.kafka.streams.TopologyTestDriver
 class GroupMapChunksTest : FunSpec({
 
   test("basic test") {
+    val streamsBuilder = StreamsBuilder()
 
     val prototypesTopicName = "prototypes-topic"
     val mapChunkUpdatesTopicName = "map-chunk-updates-topic"
 
-    val topology = groupMapChunks(
+    val topology = groupMapChunks(streamsBuilder)
-      StreamsBuilder(),
-      prototypesTopicName,
-      mapChunkUpdatesTopicName,
-    )
 
     val testDriver = TopologyTestDriver(topology)
 
-    val prototypesTestInputTopic: TestInputTopic<FactorioServerId, FactorioPrototypes> =
+    val prototypesTestInputTopic: TestInputTopic<FactorioServerId, PrototypesUpdate> =
       testDriver.createInputTopic(
         prototypesTopicName,
         jsonMapper.serde<FactorioServerId>().serializer(),
-        jsonMapper.serde<FactorioPrototypes>().serializer(),
+        jsonMapper.serde<PrototypesUpdate>().serializer(),
       )
 
     val serverId = FactorioServerId("test-server-id")
 
     prototypesTestInputTopic.pipeInput(
       serverId,
-      FactorioPrototypes(
+      PrototypesUpdate(
-        modVersion = "1.2.3",
-        tick = Tick(456u),
         listOf(
-          MapTilePrototype(
+          FactorioPrototype2.MapTile(
             PrototypeName("fake-proto"),
             layer = 1u,
             mapColour = Colour(0.3f, 0.2f, 0.1f, 1.0f),
@@ -71,26 +60,25 @@ class GroupMapChunksTest : FunSpec({
       )
     )
 
-    val mapChunkUpdatesInputTopic: TestInputTopic<FactorioServerId, FactorioEventUpdatePacket> =
+    val mapChunkUpdatesInputTopic: TestInputTopic<FactorioServerId, MapChunkUpdate> =
       testDriver.createInputTopic(
         mapChunkUpdatesTopicName,
         jsonMapper.serde<FactorioServerId>().serializer(),
-        jsonMapper.serde<FactorioEventUpdatePacket>().serializer(),
+        jsonMapper.serde<MapChunkUpdate>().serializer(),
       )
+
     mapChunkUpdatesInputTopic.pipeInput(
       serverId,
-      FactorioEventUpdatePacket(
+      MapChunkUpdate(
-        modVersion = "1.2.3",
+        key = MapChunkUpdateKey(
-        tick = Tick(854u),
-        update = MapChunkUpdate(
           MapChunkPosition(11, 22),
           SurfaceIndex(1u),
+        ),
         tileDictionary = MapTileDictionary(
           tilesXY = mapOf(
-              "1" to mapOf("2" to PrototypeKey("33"))
+            "1" to mapOf("2" to PrototypeKey(33))
           ),
-            protos = mapOf(PrototypeKey("33") to PrototypeName("fake-proto"))
+          protos = mapOf(PrototypeName("fake-proto") to PrototypeKey(33))
-          )
         )
       )
     )
@@ -98,7 +86,7 @@ class GroupMapChunksTest : FunSpec({
     val outputTopic: TestOutputTopic<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
       testDriver
         .createOutputTopic(
-          KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS,
+          TOPIC_GROUPED_MAP_CHUNKS_STATE,
           kxsBinary.serde<ServerMapChunkId>().deserializer(),
           kxsBinary.serde<ServerMapChunkTiles<ColourHex>>().deserializer(),
         )

modules/infra-factorio-server/src/.env

@@ -1,4 +1,4 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
-FACTORIO_VERSION=1.1.53
+FACTORIO_VERSION=1.1.57
-KAFKATORIO_VERSION=0.3.2
+KAFKATORIO_VERSION=0.4.0

modules/infra-kafka-cluster/src/.env

@@ -1,3 +1,3 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
-KAFKATORIO_VERSION=0.3.2
+KAFKATORIO_VERSION=0.4.0

modules/infra-kafka-cluster/src/kafka-server.properties

@@ -125,3 +125,6 @@ log.retention.check.interval.ms=300000
 ### record size ###
 message.max.bytes=10485760
 replica.fetch.max.bytes=10485760
+
+log.cleanup.policy=compact
+auto.create.topics.enable=false

modules/infra-kafka-pipe/src/.env

@@ -1,3 +1,3 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
-KAFKATORIO_VERSION=0.3.2
+KAFKATORIO_VERSION=0.4.0

modules/infra-kafka-pipe/src/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   kafka-pipe:
-    image: openjdk:11-jre-buster
+    image: dev-adamko/kafka-pipe/${KAFKATORIO_VERSION}
     container_name: "kafka-pipe"
     build:
       dockerfile: "kafka-pipe.Dockerfile"

modules/infra-kafka-pipe/src/kafka-pipe.sh

@@ -19,7 +19,7 @@ while :; do
     sed -n -e 's/^KafkatorioPacket: //p' |
     kcat -P -T \
       -b "$KAFKA_HOST" \
-      -t factorio-server-log \
+      -t kafkatorio.src.server-log \
       -k "$INSTANCE_ID" \
       -H "KAFKA-PIPE-VERSION=$KAFKATORIO_VERSION"
 

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/converters.kt

@@ -1,7 +1,7 @@
 package dev.adamko.kafkatorio.webmap
 
-import dev.adamko.kafkatorio.events.schema.MapEntityPosition
+import dev.adamko.kafkatorio.schema.common.MapEntityPosition
-import dev.adamko.kafkatorio.events.schema.PlayerUpdate
+import dev.adamko.kafkatorio.schema2.PlayerUpdate
 import io.kvision.maps.Maps
 import io.kvision.maps.externals.leaflet.geo.LatLng
 

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/json.kt

@@ -1,6 +1,6 @@
 package dev.adamko.kafkatorio.webmap
 
-import dev.adamko.kafkatorio.events.schema.jsonMapperKafkatorio
+import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
 import kotlinx.serialization.json.Json
 
 val jsonMapper = Json {

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/state.kt

@@ -1,12 +1,12 @@
 package dev.adamko.kafkatorio.webmap
 
-import dev.adamko.kafkatorio.events.schema.Colour
+import dev.adamko.kafkatorio.schema.common.Colour
-import dev.adamko.kafkatorio.events.schema.MapEntityPosition
+import dev.adamko.kafkatorio.schema.common.MapEntityPosition
-import dev.adamko.kafkatorio.events.schema.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.PlayerIndex
-import dev.adamko.kafkatorio.events.schema.PlayerUpdate
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
-import dev.adamko.kafkatorio.events.schema.SurfaceIndex
+import dev.adamko.kafkatorio.schema.common.Tick
-import dev.adamko.kafkatorio.events.schema.Tick
+import dev.adamko.kafkatorio.schema.common.toHexString
-import dev.adamko.kafkatorio.events.schema.converters.toHexString
+import dev.adamko.kafkatorio.schema2.PlayerUpdate
 import io.kvision.maps.Maps
 import io.kvision.maps.Maps.Companion.L
 import io.kvision.maps.externals.leaflet.geo.LatLng
@@ -43,8 +43,8 @@ fun factorioGameReducer(state: FactorioGameState, update: FactorioUpdate): Facto
   when (update) {
     is FactorioUpdate.Player -> {
 
-      val playerState = state.players.getOrElse(update.data.index) {
+      val playerState = state.players.getOrElse(update.data.key.index) {
-        PlayerState(update.data.index, update.tick)
+        PlayerState(update.data.key.index, update.tick)
       }
 
       playerState.update(update.tick, update.data)

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt

@@ -1,13 +1,14 @@
 package dev.adamko.kafkatorio.webmap
 
-import dev.adamko.kafkatorio.events.schema.EntityUpdate
+import dev.adamko.kafkatorio.schema2.ConfigurationUpdate
-import dev.adamko.kafkatorio.events.schema.FactorioConfigurationUpdate
+import dev.adamko.kafkatorio.schema2.ConsoleChatUpdate
-import dev.adamko.kafkatorio.events.schema.FactorioEvent
+import dev.adamko.kafkatorio.schema2.ConsoleCommandUpdate
-import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
+import dev.adamko.kafkatorio.schema2.EntityUpdate
-import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
+import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
-import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
+import dev.adamko.kafkatorio.schema2.MapChunkUpdate
-import dev.adamko.kafkatorio.events.schema.MapChunkUpdate
+import dev.adamko.kafkatorio.schema2.PlayerUpdate
-import dev.adamko.kafkatorio.events.schema.PlayerUpdate
+import dev.adamko.kafkatorio.schema2.PrototypesUpdate
+import dev.adamko.kafkatorio.schema2.SurfaceUpdate
 import io.kvision.redux.ReduxStore
 import kotlinx.serialization.decodeFromString
 import org.w3c.dom.MessageEvent
@@ -35,26 +36,23 @@ class WebsocketService(
 
       println(data.replace('\n', ' '))
 
-      when (val event: KafkatorioPacket = jsonMapper.decodeFromString(data)) {
+      val packet: KafkatorioPacket2 = jsonMapper.decodeFromString(data)
-        is FactorioEvent             -> {
+
-        }
+      when (val packetData = packet.data) {
-        is FactorioEventUpdatePacket -> {
+
-          when (val eventUpdate = event.update) {
+        is PlayerUpdate   -> reduxStore.dispatch(FactorioUpdate.Player(packet.tick, packetData))
-            is PlayerUpdate   -> reduxStore.dispatch(FactorioUpdate.Player(event.tick, eventUpdate))
+        is ConfigurationUpdate,
+        is ConsoleChatUpdate,
+        is ConsoleCommandUpdate,
+        is PrototypesUpdate,
+        is SurfaceUpdate,
         is EntityUpdate,
         is MapChunkUpdate -> {
-            }
-          }
-        }
-        is FactorioConfigurationUpdate,
-        is FactorioPrototypes        -> {
           // to be continued...
         }
       }
-
     } else {
       println(data)
     }
-
   }
 }