Merge branch 'main' of github.com:adamko-dev/kafkatorio

build.gradle.kts

@@ -12,7 +12,7 @@ plugins {
 }
 
 group = "dev.adamko.kafkatorio"
-version = "0.6.0"
+version = "0.6.1"
 
 val licenseFile: RegularFile by extra(layout.projectDirectory.file("LICENSE"))
 

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPortalPublishClient.kt

@@ -1,4 +1,4 @@
-package dev.adamko.kafkatorio.factoriomod
+package dev.adamko.kafkatorio.factoriomod.portal
 
 import io.ktor.client.HttpClient
 import io.ktor.client.engine.cio.CIO

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPublishTask.kt

@@ -1,5 +1,6 @@
-package dev.adamko.kafkatorio.factoriomod
+package dev.adamko.kafkatorio.factoriomod.portal
 
+import dev.adamko.kafkatorio.factoriomod.FactorioMod
 import io.ktor.http.URLBuilder
 import io.ktor.http.takeFrom
 import javax.inject.Inject
@@ -59,7 +60,7 @@ abstract class FactorioModPublishTask @Inject constructor(
 
 
   init {
-    group = "factorio"
+    group = FactorioMod.TASK_GROUP
   }
 
 

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/model.kt

@@ -1,4 +1,4 @@
-package dev.adamko.kafkatorio.factoriomod
+package dev.adamko.kafkatorio.factoriomod.portal
 
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/tasks.kt

@@ -0,0 +1,12 @@
+package dev.adamko.kafkatorio.factoriomod
+
+object FactorioMod {
+
+  const val TASK_GROUP = "factorioMod"
+
+  /** Lifecycle task for publishing the mod locally */
+  const val PUBLISH_MOD_LOCAL_TASK_NAME = "publishModLocal"
+
+  const val PUBLISH_MOD_TASK_NAME = "publishMod"
+
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts

@@ -12,16 +12,8 @@ val projectJvmTarget = "11"
 
 
 dependencies {
-
-//  implementation(platform("org.jetbrains.kotlin:kotlin-bom"))
-
-//  testImplementation(platform("org.junit:junit-bom:${Versions.junit}"))
   testImplementation("org.junit.jupiter:junit-jupiter")
-//  testRuntimeOnly("org.junit.platform:junit-platform-launcher") {
-//    because("Only needed to run tests in a version of IntelliJ IDEA that bundles older versions")
-//  }
 
-//  testImplementation(platform("io.kotest:kotest-bom:${Versions.kotest}"))
   testImplementation("io.kotest:kotest-runner-junit5")
   testImplementation("io.kotest:kotest-assertions-core")
   testImplementation("io.kotest:kotest-property")

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypeScriptTask.kt

@@ -8,7 +8,7 @@ import org.gradle.api.tasks.OutputDirectory
 import org.gradle.api.tasks.TaskAction
 
 
-abstract class GenerateTypescriptTask @Inject constructor(
+abstract class GenerateTypeScriptTask @Inject constructor(
   private val fileOps: FileSystemOperations,
 ) : JavaExec() {
 

gradle/libs.versions.toml

@@ -35,6 +35,8 @@ ktor                  = "2.0.2"
 
 hoplite               = "2.1.5"
 
+simpleSyslogParser    = "0.0.6"
+
 kotkaStreams          = "dev-SNAPSHOT"
 
 ## test ##
@@ -156,10 +158,12 @@ ktor-serverCallLogging        = { group = "io.ktor",  name = "ktor-server-call-l
 ktor-serverCallId             = { group = "io.ktor",  name = "ktor-server-call-id"             }
 
 ktor-network                  = { group = "io.ktor",  name = "ktor-network"                    }
+ktor-networkTls               = { group = "io.ktor",  name = "ktor-network-tls"                }
 
 ktor-test-clientMock          = { group = "io.ktor",  name = "ktor-client-mock"                }
 ktor-test-serverHost          = { group = "io.ktor",  name = "ktor-server-test-host"           }
 
+
 ### jackson ##
 #jackson-bom     = { group = " com.fasterxml.jackson.module",  name = ":jackson-module-kotlin", version = "2.13.+"     }
 #jackson-kotlin     = { group = " com.fasterxml.jackson.module",  name = ":jackson-module-kotlin", version = "2.13.+"  }
@@ -205,6 +209,10 @@ hoplite-core     = { group = "com.sksamuel.hoplite", name = "hoplite-core",
 hoplite-datetime = { group = "com.sksamuel.hoplite", name = "hoplite-datetime", version.ref = "hoplite" }
 hoplite-yaml     = { group = "com.sksamuel.hoplite", name = "hoplite-yaml",     version.ref = "hoplite" }
 
+
+simpleSyslogParser = { group = "com.github.palindromicity", name = "simple-syslog", version.ref = "simpleSyslogParser" }
+
+
 #############
 ### test  ###
 #############

modules/events-mod/build.gradle.kts

@@ -1,4 +1,4 @@
-import dev.adamko.kafkatorio.factoriomod.FactorioModPublishTask
+import dev.adamko.kafkatorio.factoriomod.portal.FactorioModPublishTask
 import dev.adamko.kafkatorio.gradle.asConsumer
 import dev.adamko.kafkatorio.gradle.asProvider
 import dev.adamko.kafkatorio.gradle.dropDirectory
@@ -39,11 +39,11 @@ val projectTokens: MapProperty<String, String> =
     ?: error("error getting projectTokens")
 //val projectTokens: MapProperty<String, String> by rootProject.extra
 
-val projectTokensX = projectTokens.apply {
+val projectTokensX1 = projectTokens.apply {
   put("mod.name", modName)
   put("mod.title", "Kafkatorio Events")
   put("mod.description", modDescription)
-  put("factorio.version", modFactorioCompatibility.get())
+  put("factorio.version", modFactorioCompatibility)
 }
 
 val tsSrcDir: Directory = layout.projectDirectory.dir("src/main/typescript")
@@ -82,7 +82,7 @@ val installEventsTsSchema by tasks.registering(Sync::class) {
   dependsOn(typescriptEventsSchema)
 //  dependsOn(fixLink)
 
-  val outputDir = layout.projectDirectory.dir("src/main/typescript/generated/kafkatorio-schema")
+  val outputDir = layout.projectDirectory.dir("src/main/typescript/generated/")
   outputs.dir(outputDir)
 
   from(
@@ -107,16 +107,16 @@ val installEventsTsSchema by tasks.registering(Sync::class) {
 val zipNameProvider = provider { distributionZipName }
 
 tasks.distZip {
-  val projectTokensXX = projectTokensX
+  val projectTokensXX22 = projectTokensX1
 
   inputs.property("zipNameProvider", zipNameProvider)
-  inputs.property("projectTokens", projectTokensXX)
+  inputs.property("projectTokens", projectTokensXX22)
 
   archiveFileName.set(zipNameProvider)
 }
 
 
-val publishMod by tasks.registering(FactorioModPublishTask::class) {
+val publishModToPortal by tasks.registering(FactorioModPublishTask::class) {
   dependsOn(tasks.check)
 
   distributionZip.set(tasks.distZip.flatMap { it.archiveFile })
@@ -131,6 +131,7 @@ distributions {
   main {
 
     distributionBaseName.set(modName)
+    val tokens333: MutableMap<String, String> = projectTokensX1.get()
 
     contents {
       from(layout.projectDirectory.dir("src/main/resources/mod-data")) {
@@ -139,7 +140,7 @@ distributions {
       from(licenseFile)
       from(typescriptToLua.map { it.outputDirectory })
       filesNotMatching("**/*.png") {
-        filter<ReplaceTokens>("tokens" to projectTokens.get())
+        filter<ReplaceTokens>("tokens" to tokens333)
       }
       includeEmptyDirs = false
       exclude {
@@ -154,8 +155,8 @@ distributions {
 
 
 tasks.withType<Zip>().configureEach {
-  val projectTokensXX = projectTokensX
+  val projectTokensXX444 = projectTokensX1
-  inputs.property("projectTokens", projectTokensXX)
+  inputs.property("projectTokens", projectTokensXX444)
 }
 
 

modules/events-mod/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

modules/events-mod/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafkatorio-events-mod",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "private": true,
   "scripts": {
     "build": "tstl",
@@ -9,7 +9,8 @@
   "engines": {
     "node": "~14.19.1"
   },
-  "devDependencies": {},
+  "devDependencies": {
+  },
   "dependencies": {
     "lua-types": "^2.11.0",
     "typescript-to-lua": "1.6.0",
@@ -17,7 +18,10 @@
     "typescript": "4.7.4"
   },
   "license": "UNLICENSED",
-  "peerDependencies": {},
+  "peerDependencies": {
-  "optionalDependencies": {},
+  },
-  "bundledDependencies": []
+  "optionalDependencies": {
+  },
+  "bundledDependencies": [
+  ]
 }

modules/events-mod/src/main/typescript/control.ts

@@ -1,23 +1,11 @@
-import {emitConfigurationUpdate} from "./main/config-update/handleConfigurationUpdate";
-import {emitPrototypes} from "./main/config-update/prototypes";
 import {initGlobal} from "./main/global-init";
-import KafkatorioSettings from "./main/settings/KafkatorioSettings";
+import {emitPrototypes} from "./main/events/prototypeUpdates";
 
 require("./main/events/registerHandlers")
-require("./main/events/onMapChunkEventListeners")
+require("./main/events/mapChunkUpdates")
-require("./main/events/onPlayerEventListeners")
+require("./main/events/playerUpdates")
-require("./main/events/EntityEvents")
+require("./main/events/entityUpdates")
 require("./main/commands/kafkatorioCommands")
-require("./main/cache/cacheChecker")
-
-
-script.on_configuration_changed((data: ConfigurationChangedData) => {
-  // runs whenever the version of the base game, or a mod, changes
-  initGlobal(true)
-  // KafkatorioSettings.loadSettings()
-  emitConfigurationUpdate(data)
-  emitPrototypes()
-})
 
 
 script.on_init(() => {

modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts

@@ -13,6 +13,7 @@ export type KafkatorioPacketData =
   | KafkatorioPacketData.ConsoleChatUpdate
   | KafkatorioPacketData.ConsoleCommandUpdate
   | KafkatorioPacketData.EntityUpdate
+  | KafkatorioPacketData.Error
   | KafkatorioPacketData.MapChunkUpdate
   | KafkatorioPacketData.PlayerUpdate
   | KafkatorioPacketData.PrototypesUpdate
@@ -28,6 +29,7 @@ export namespace KafkatorioPacketData {
     EntityUpdate = "kafkatorio.packet.keyed.EntityUpdate",
     MapChunkUpdate = "kafkatorio.packet.keyed.MapChunkUpdate",
     PlayerUpdate = "kafkatorio.packet.keyed.PlayerUpdate",
+    Error = "kafkatorio.packet.KafkatorioPacketData.Error",
   }
   
   export interface ConfigurationUpdate {
@@ -115,6 +117,12 @@ export namespace KafkatorioPacketData {
     disconnectReason?: string | null;
     isRemoved?: boolean | null;
   }
+  
+  export interface Error {
+    type: KafkatorioPacketData.Type.Error;
+    message?: string | null;
+    rawValue?: string | null;
+  }
 }
 
 export type UInt = uint;
@@ -240,12 +248,12 @@ export type List = any;
 export type PrototypeKey = Int;
 
 export interface MinedProduct {
-  type: Type;
+  type: MinedProductType;
   name: PrototypeName;
   amount: Double | null;
 }
 
-export enum Type {
+export enum MinedProductType {
   item = "item",
   fluid = "fluid",
 }

modules/events-mod/src/main/typescript/main/PacketEmitter.ts

@@ -1,9 +1,10 @@
 import {
   KafkatorioPacket,
   KafkatorioPacketData
-} from "../generated/kafkatorio-schema/kafkatorio-schema";
+} from "../generated/kafkatorio-schema";
 import {KafkatorioKeyedPacketData} from "./types";
 
+
 export class KafkatorioPacketEmitter {
 
   emitInstantPacket<DATA extends KafkatorioPacketData>(

modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts

@@ -1,5 +1,6 @@
 import KafkatorioSettings from "../settings/KafkatorioSettings";
 import {KafkatorioKeyedPacketData} from "../types";
+import packetEmitter from "../PacketEmitter";
 
 export namespace EventUpdates {
 
@@ -71,6 +72,10 @@ export namespace EventUpdates {
       if (expirationDurationTicks != undefined) {
         entry.expirationDurationTicks = expirationDurationTicks
       }
+
+      if (entry.expirationDurationTicks != undefined && entry.expirationDurationTicks <= 0) {
+        this.extractAndEmitExpiredPackets()
+      }
     }
 
 
@@ -90,8 +95,9 @@ export namespace EventUpdates {
     }
 
 
+    /** Removes all expired entries from the cache, and return them. */
     // use in/out v4.7.0 https://github.com/microsoft/TypeScript/pull/48240
-    public extractExpired(): Array<KafkatorioKeyedPacketData> {
+    private extractExpired(): Array<KafkatorioKeyedPacketData> {
       let countExpired = 0
       let countTotal = 0
       const expiredData: Array<KafkatorioKeyedPacketData> = []
@@ -114,6 +120,25 @@ export namespace EventUpdates {
     }
 
 
+    /**
+     * Removes and emits all expired packets.
+     *
+     * @return the number of emitted packets
+     */
+    public extractAndEmitExpiredPackets() {
+      const cachedEvents: Array<KafkatorioKeyedPacketData> = this.extractExpired()
+
+      if (cachedEvents.length > 0) {
+        log(`[EventDataCache] emitting ${cachedEvents.length} events on tick ${game.tick}`)
+        for (const event of cachedEvents) {
+          packetEmitter.emitKeyedPacket(event)
+        }
+      }
+
+      return cachedEvents.length
+    }
+
+
     private getCacheEntry<PACKET extends KafkatorioKeyedPacketData>(
         key: EventUpdates.PacketKey<PACKET>,
         type: EventUpdates.PacketType<PACKET>,

modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts

@@ -1,16 +0,0 @@
-import EventCacheService from "./EventDataCache";
-import packetEmitter from "../PacketEmitter";
-import {KafkatorioKeyedPacketData} from "../types";
-
-/** Every 0.1 seconds */
-script.on_nth_tick(6, (nthTick: NthTickEventData) => {
-  const cachedEvents: Array<KafkatorioKeyedPacketData> = EventCacheService.extractExpired()
-
-  if (cachedEvents.length > 0) {
-    log(`nth tick ${nthTick.tick} has ${cachedEvents.length} events`)
-
-    for (const event of cachedEvents) {
-      packetEmitter.emitKeyedPacket(event)
-    }
-  }
-})

modules/events-mod/src/main/typescript/main/commands/kafkatorioCommands.ts

@@ -1,7 +1,6 @@
-import {emitPrototypes} from "../config-update/prototypes";
 import {initGlobal} from "../global-init";
-import floor = math.floor;
 import {Queue} from "../queue/queue";
+import {emitPrototypes} from "../events/prototypeUpdates";
 
 
 commands.add_command(
@@ -32,8 +31,8 @@ commands.add_command(
             let player = game.players[e.player_index]
 
             const chunkPosition: ChunkPosition = {
-              x: floor(player.position.x / 32),
+              x: math.floor(player.position.x / 32),
-              y: floor(player.position.y / 32),
+              y: math.floor(player.position.y / 32),
             }
 
             const chunkXMin = chunkPosition.x - radius
@@ -56,7 +55,7 @@ commands.add_command(
                     position: {x: chunk.x, y: chunk.y},
                     area: chunk.area,
                     surface: surface,
-                    tick: e.tick
+                    tick: e.tick,
                   }
 
                   Queue.enqueue(

modules/events-mod/src/main/typescript/main/events/configurationUpdate.ts

@@ -3,11 +3,23 @@ import {
   ConfigurationUpdateGameData,
   ConfigurationUpdateModData,
   KafkatorioPacketData
-} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+} from "../../generated/kafkatorio-schema";
+import {initGlobal} from "../global-init";
+import {emitPrototypes} from "./prototypeUpdates";
 import Type = KafkatorioPacketData.Type;
 import ConfigurationUpdate = KafkatorioPacketData.ConfigurationUpdate;
 
-export function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
+
+script.on_configuration_changed((data: ConfigurationChangedData) => {
+  // runs whenever the version of the base game, or a mod, changes
+  initGlobal(true)
+  // KafkatorioSettings.loadSettings()
+  emitConfigurationUpdate(data)
+  emitPrototypes()
+})
+
+
+function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
 
   const configUpdateData: ConfigurationUpdate = {
     type: Type.ConfigurationUpdate,
@@ -20,6 +32,7 @@ export function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
   PacketEmitter.emitInstantPacket(configUpdateData)
 }
 
+
 function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdateGameData {
   return {
     oldVersion: changeData.old_version ?? null,
@@ -27,6 +40,7 @@ function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdate
   }
 }
 
+
 function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] {
 
   let namesOfChangedMods: string[] = Object.keys(e.mod_changes)
@@ -38,7 +52,7 @@ function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] {
 
   return modNames.map((modName) => {
 
-    let currentVer = script.active_mods[modName] ?? e.mod_changes[modName].new_version ?? null
+    let currentVer = script.active_mods[modName] ?? e.mod_changes[modName]?.new_version ?? null
     let previousVer = e.mod_changes[modName]?.old_version ?? null
 
     return {

modules/events-mod/src/main/typescript/main/events/consoleUpdates.ts

@@ -1,7 +1,8 @@
 import PacketEmitter from "../PacketEmitter";
-import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import {KafkatorioPacketData} from "../../generated/kafkatorio-schema";
 import Type = KafkatorioPacketData.Type;
 
+
 script.on_event(
     defines.events.on_console_chat,
     (event: OnConsoleChatEvent) => {
@@ -15,6 +16,7 @@ script.on_event(
     }
 )
 
+
 script.on_event(
     defines.events.on_console_command,
     (event: OnConsoleCommandEvent) => {

modules/events-mod/src/main/typescript/main/events/converters.ts

@@ -1,5 +1,5 @@
 import {EventName} from "../types";
-import {Colour, MapChunkPosition} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import {Colour, MapChunkPosition} from "../../generated/kafkatorio-schema";
 
 export namespace Converters {
 
@@ -57,4 +57,13 @@ export namespace Converters {
     return [position.x, position.y]
   }
 
+
+  export function convertCollisionMaskToNames(cm: CollisionMask): string[] {
+    let masks: string[] = []
+    for (let [name] of pairs(cm)) {
+      masks.push(name)
+    }
+    return masks
+  }
+
 }

modules/events-mod/src/main/typescript/main/events/entityUpdates.ts

@@ -3,7 +3,7 @@ import EventDataCache from "../cache/EventDataCache";
 import {
   EntityUpdateKey,
   KafkatorioPacketData
-} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+} from "../../generated/kafkatorio-schema";
 
 
 script.on_event(defines.events.on_built_entity, (e: OnBuiltEntityEvent) => {

modules/events-mod/src/main/typescript/main/events/mapChunkUpdates.ts

@@ -4,7 +4,7 @@ import EventUpdatesManager, {EventUpdates} from "../cache/EventDataCache";
 import {
   KafkatorioPacketData,
   MapChunkPosition
-} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+} from "../../generated/kafkatorio-schema";
 import CacheKey = EventUpdates.PacketKey;
 import Type = KafkatorioPacketData.Type;
 

modules/events-mod/src/main/typescript/main/events/playerUpdates.ts

@@ -1,7 +1,11 @@
 import {Converters} from "./converters";
 import EventUpdatesManager from "../cache/EventDataCache";
-import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import {
+  KafkatorioPacketData,
+  PlayerUpdateKey
+} from "../../generated/kafkatorio-schema";
 import Type = KafkatorioPacketData.Type;
+import packetEmitter from "../PacketEmitter";
 
 
 type PlayerUpdater = (player: LuaPlayer, data: KafkatorioPacketData.PlayerUpdate) => void
@@ -44,7 +48,7 @@ function playerUpdateThrottle(
         data.events[eventName] ??= []
         data.events[eventName].push(event.tick)
       },
-      expirationDurationTicks
+      expirationDurationTicks,
   )
 }
 
@@ -183,13 +187,26 @@ script.on_event(
     defines.events.on_pre_player_left_game,
     (event: OnPrePlayerLeftGameEvent) => {
       log(`on_pre_player_left_game ${event.tick} ${event.name}`)
-      playerUpdateImmediate(
+
-          event,
+      const playerUpdateKey: PlayerUpdateKey = {
-          (player, data) => {
+        index: event.player_index
-            data.disconnectReason = disconnectReasons.get(event.reason)
-            playerOnlineInfo(player, data)
       }
-      )
+
+      const playerUpdate: KafkatorioPacketData.PlayerUpdate = {
+        type: KafkatorioPacketData.Type.PlayerUpdate,
+        key: playerUpdateKey,
+        disconnectReason: disconnectReasons.get(event.reason)
+      }
+
+      packetEmitter.emitKeyedPacket(playerUpdate)
+
+      // playerUpdateImmediate(
+      //     event,
+      //     (player, data) => {
+      //       data.disconnectReason = disconnectReasons.get(event.reason)
+      //       playerOnlineInfo(player, data)
+      //     }
+      // )
     }
 )
 

modules/events-mod/src/main/typescript/main/events/prototypeUpdates.ts

@@ -1,24 +1,21 @@
-import {Converters} from "../events/converters";
 import PacketEmitter from "../PacketEmitter";
-import {
+import {FactorioPrototype, KafkatorioPacketData} from "../../generated/kafkatorio-schema";
-  FactorioPrototype,
+import {Converters} from "./converters";
-  KafkatorioPacketData
-} from "../../generated/kafkatorio-schema/kafkatorio-schema";
-import Type = KafkatorioPacketData.Type;
 import PrototypesUpdate = KafkatorioPacketData.PrototypesUpdate;
 
 
 export function emitPrototypes() {
 
   const data: PrototypesUpdate = {
-    prototypes: prototypes(),
+    prototypes: prototypeUpdates(),
-    type: Type.PrototypesUpdate,
+    type: KafkatorioPacketData.Type.PrototypesUpdate,
   }
 
   PacketEmitter.emitInstantPacket(data)
 }
 
-function prototypes(): FactorioPrototype[] {
+
+function prototypeUpdates(): FactorioPrototype[] {
   let prototypes: FactorioPrototype[] = []
 
   prototypes.push(...getMapTilePrototypes())
@@ -26,6 +23,7 @@ function prototypes(): FactorioPrototype[] {
   return prototypes
 }
 
+
 function getMapTilePrototypes(): FactorioPrototype.MapTile[] {
   let tiles: FactorioPrototype.MapTile[] = []
   for (let [, tile] of game.tile_prototypes) {
@@ -35,7 +33,7 @@ function getMapTilePrototypes(): FactorioPrototype.MapTile[] {
       name: tile.name,
       order: tile.order,
       layer: tile.layer,
-      collisionMasks: convertCollisionMaskToNames(tile.collision_mask),
+      collisionMasks: Converters.convertCollisionMaskToNames(tile.collision_mask),
       mapColour: Converters.mapColour(tile.map_color),
       canBeMined: tile.mineable_properties.minable,
     })
@@ -60,11 +58,3 @@ function getMapTilePrototypes(): FactorioPrototype.MapTile[] {
 //   }
 //   return tiles
 // }
-
-function convertCollisionMaskToNames(cm: CollisionMask): string[] {
-  let masks: string[] = []
-  for (let [name] of pairs(cm)) {
-    masks.push(name)
-  }
-  return masks
-}

modules/events-mod/src/main/typescript/main/events/registerHandlers.ts

@@ -1,34 +1,11 @@
 import {Queue} from "../queue/queue";
 import {isEventType} from "./eventTypeCheck";
 import {Converters} from "./converters";
-import KafkatorioSettings from "../settings/KafkatorioSettings";
+import {handleChunkGeneratedEvent} from "./mapChunkUpdates";
-import {handleChunkGeneratedEvent} from "./onMapChunkEventListeners";
+import {handleSurfaceUpdate} from "./surfaceUpdates";
-import {handleSurfaceUpdate} from "./handlers";
+import EventCacheService from "../cache/EventDataCache";
 
 
-// script.on_event(
-//     defines.events.on_player_mined_entity,
-//     (e: OnPlayerMinedEntityEvent) => {
-//       let eventName = Converters.eventNameString(e.name)
-//       // handlePlayerUpdate(e.tick, eventName, e.player_index)
-//       handleEntityUpdate(e.tick, eventName, e.entity)
-//     }
-// )
-
-// script.on_event(
-//     defines.events.on_player_joined_game,
-//     (e: OnPlayerJoinedGameEvent) => {
-//       handlePlayerUpdate(e.tick, mapEventIdToName.get(e.name), e.player_index)
-//     }
-// )
-
-// script.on_event(
-//     defines.events.on_player_changed_position,
-//     (e: OnPlayerChangedPositionEvent) => {
-//       handlePlayerUpdate(e.tick, mapEventIdToName.get(e.name), e.player_index)
-//     }
-// )
-
 script.on_event(
     defines.events.on_tick,
     (event: OnTickEvent) => {
@@ -38,18 +15,26 @@ script.on_event(
         }
       }
 
+      if (event.tick % 6 == 0) {
+        // Every 0.1 seconds
+        const emittedCount = EventCacheService.extractAndEmitExpiredPackets()
+        if (emittedCount > 0) {
+          log(`[on_tick:${event.tick}] emitted ${emittedCount} events`)
+        }
+      }
+
       if (event.tick % 7 == 0) {
         const events: EventData[] = Queue.dequeueValues(1)
 
         if (events.length > 0) {
-          log(`[${event.tick}] dequed ${events.length} events, current size: ${Queue.size()}`)
+          log(`[on_tick:${event.tick}] dequeued ${events.length} events, current size: ${Queue.size()}`)
 
           let i = 1
           for (const event of events) {
             if (isEventType(event, defines.events.on_chunk_generated)) {
               let eName = Converters.eventNameString(event.name)
 
-              log(`[${event.tick}] dequed event ${eName}, delay ${i}`)
+              log(`[on_tick:${event.tick}] dequed event ${eName}, delay ${i}`)
               handleChunkGeneratedEvent(event, i)
               i += 1
             }
@@ -58,12 +43,3 @@ script.on_event(
       }
     }
 )
-
-
-script.on_event(
-    defines.events.on_runtime_mod_setting_changed,
-    (e: OnRuntimeModSettingChangedEvent) => {
-      print(`Settings changed ${e.setting_type}, ${e.player_index}, ${e.setting}, ${e.mod_name}`)
-      // KafkatorioSettings.loadSettings()
-    }
-)

modules/events-mod/src/main/typescript/main/events/settingsUpdate.ts

@@ -0,0 +1,7 @@
+script.on_event(
+    defines.events.on_runtime_mod_setting_changed,
+    (e: OnRuntimeModSettingChangedEvent) => {
+      print(`Settings changed ${e.setting_type}, ${e.player_index}, ${e.setting}, ${e.mod_name}`)
+      // KafkatorioSettings.loadSettings()
+    }
+)

modules/events-mod/src/main/typescript/main/events/surfaceUpdates.ts

@@ -1,5 +1,5 @@
 import PacketEmitter from "../PacketEmitter";
-import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema";
+import {KafkatorioPacketData} from "../../generated/kafkatorio-schema";
 import Type = KafkatorioPacketData.Type;
 
 

modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts

@@ -11,35 +11,12 @@ export class KafkatorioSettingsConfig {
   }
 
 
-  // private eventCacheExpirationTicks: Record<KafkatorioKeyedPacketTypes, double> = {
-  //   "kafkatorio.packet.keyed.PlayerUpdate":
-  //       KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.PlayerUpdate],
-  // "kafkatorio.packet.keyed.MapChunkUpdate":
-  // KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.MapChunkUpdate],
-  // "kafkatorio.packet.keyed.EntityUpdate":
-  // KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.EntityUpdate],
-  // }
-
   public getEventCacheExpirationTicks(packet: KafkatorioKeyedPacketData): uint {
     const settingName = KafkatorioSettingsConfig.cacheDurationSettingName(packet.type)
     const seconds: double = settings.global[settingName].value as double
     return seconds * 60 // convert ticks to seconds
   }
 
-  // private static types: KafkatorioKeyedPacketTypes[] = [
-  //   KafkatorioPacketData.Type.PlayerUpdate,
-  //   KafkatorioPacketData.Type.MapChunkUpdate,
-  //   KafkatorioPacketData.Type.EntityUpdate,
-  // ]
-
-  // public loadSettings(): void {
-  //   for (const type of KafkatorioSettingsConfig.types) {
-  //     const settingName = KafkatorioSettingsConfig.cacheDurationSettingName(type)
-  //     const seconds: double = settings.global[settingName].value as double
-  //     this.eventCacheExpirationTicks[type] = seconds * 60 // convert ticks to seconds
-  //   }
-  // }
-
 
   public initialiseSettings(data: Data): void {
     for (const [type,] of pairs(KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds)) {

modules/events-mod/src/main/typescript/main/types.d.ts

@@ -1,4 +1,4 @@
-import {KafkatorioPacketData} from "../generated/kafkatorio-schema/kafkatorio-schema";
+import {KafkatorioPacketData} from "../generated/kafkatorio-schema";
 
 export type EventName = keyof typeof defines.events
 

modules/events-schema/build.gradle.kts

@@ -1,6 +1,6 @@
 import dev.adamko.kafkatorio.gradle.asProvider
 import dev.adamko.kafkatorio.gradle.typescriptAttributes
-import dev.adamko.kafkatorio.task.GenerateTypescriptTask
+import dev.adamko.kafkatorio.task.GenerateTypeScriptTask
 
 
 plugins {
@@ -14,25 +14,8 @@ plugins {
 kotlin {
 
   js(IR) {
-//    binaries.executable()
     browser {}
-    // browser{} causes this error:
+    // this is a library - don't set binaries.executable()
-    // Cannot load "webpack", it is not registered!
-    // Perhaps you are missing some plugin?
-    // Cannot load "sourcemap", it is not registered!
-    // Perhaps you are missing some plugin?
-    // Server start failed on port 9876: Error: No provider for "framework:mocha"! (Resolving: framework:mocha)
-    // browser {
-    //   runTask {
-    //     sourceMaps = true
-    //   }
-    //   testTask {
-    //     useKarma {
-    //       useChromeHeadless()
-    //     }
-    //   }
-    // }
-//    nodejs()
   }
   jvm {
     val main by compilations.getting {
@@ -75,9 +58,6 @@ kotlin {
 
         implementation(project.dependencies.platform(libs.kotest.bom))
         implementation("io.kotest:kotest-framework-engine")
-//        implementation("io.kotest:kotest-assertions-core")
-//        implementation("io.kotest:kotest-property")
-//        implementation("io.kotest:kotest-assertions-json")
         implementation(libs.kotest.core)
         implementation(libs.kotest.datatest)
         implementation(libs.kotest.prop)
@@ -88,10 +68,7 @@ kotlin {
     val jvmMain by getting {
       dependencies {
         implementation("com.github.aSemy:ts-generator:v1.2.1")
-//        implementation("com.github.ntrrgc:ts-generator:1.1.2")
         implementation(kotlin("reflect"))
-
-//        implementation("com.github.adamko-dev:kotlinx-serialization-typescript-generator:0.0.5")
       }
     }
 
@@ -114,7 +91,7 @@ val mainCompilation: Provider<FileCollection> =
   kotlin.jvm().compilations.named("main").map { it.runtimeDependencyFiles }
 
 
-val generateTypescript by tasks.registering(GenerateTypescriptTask::class) {
+val generateTypeScript by tasks.registering(GenerateTypeScriptTask::class) {
   dependsOn(jvmJar)
   classpath(
     jvmJar,
@@ -125,68 +102,26 @@ val generateTypescript by tasks.registering(GenerateTypescriptTask::class) {
   args(temporaryDir.canonicalPath)
 }
 
-//val generateTypescript2 by tasks.registering(JavaExec::class) {
-//  group = "kt2ts"
-//
-//  dependsOn(jvmJar)
-//
-//  inputs.files(mainCompilation)
-//
-//  classpath(
-//    jvmJar,
-//    mainCompilation,
-//  )
-//  mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2ts2Kt")
-//
-//  args(temporaryDir)
-//
-//  val buildOutput = layout.buildDirectory.dir("generated/typescript")
-//  outputs.dir(buildOutput)
-//
-//  doFirst {
-//    delete(temporaryDir)
-//    mkdir(temporaryDir)
-//  }
-//
-//  doLast {
-//    sync {
-//      from(temporaryDir)
-//      into(buildOutput)
-//      include("**/*.ts")
-//    }
-//  }
-//}
-
 val schemaTsDistributionName: Provider<String> = providers.provider {
   "${rootProject.name}-${project.name}"
 }
 
-val generateTypescriptOutputFiles: Provider<FileTree> =
+val generateTypeScriptOutputFiles: Provider<FileTree> =
-  generateTypescript.map { it.outputs.files.asFileTree }
+  generateTypeScript.map { it.outputs.files.asFileTree }
 
 val schemaTs by distributions.registering {
   distributionBaseName.set(schemaTsDistributionName)
   contents {
-    from(generateTypescriptOutputFiles)
+    from(generateTypeScriptOutputFiles)
   }
 }
 
 val schemaTsZipTask: TaskProvider<Zip> = tasks.named<Zip>("${schemaTs.name}DistZip")
 val schemaTsZipTaskArchiveFile: Provider<RegularFile> = schemaTsZipTask.flatMap { it.archiveFile }
 
-val typescriptModelGenerated: Configuration by configurations.creating {
+val typeScriptModelGenerated: Configuration by configurations.creating {
   asProvider()
   typescriptAttributes(objects)
 
   outgoing.artifact(schemaTsZipTaskArchiveFile)
 }
-
-
-//tasks.matching {
-//  it.name == "jsGenerateExternalsIntegrated"
-//}.configureEach {
-//  @Suppress("UnstableApiUsage")
-//  notCompatibleWithConfigurationCache("try to prevent 'Projects must be configuring' error")
-////  outputs.upToDateWhen { true }
-//  dependsOn(":kotlinNpmInstall")
-//}

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt

@@ -5,6 +5,7 @@ import kotlinx.serialization.Serializable
 
 
 @Serializable
+@SerialName("kafkatorio.resource.MinedProduct")
 data class MinedProduct(
   /** `item` or `fluid`. */
   val type: Type,
@@ -30,6 +31,7 @@ data class MinedProduct(
 ) {
 
   @Serializable
+  @SerialName("kafkatorio.resource.MinedProductType")
   enum class Type {
     @SerialName("item")
     ITEM,

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt

@@ -16,4 +16,13 @@ data class KafkatorioPacket(
 
 @Serializable
 @SerialName("kafkatorio.packet.KafkatorioPacketData")
-sealed class KafkatorioPacketData
+sealed class KafkatorioPacketData {
+
+  @Serializable
+  @SerialName("kafkatorio.packet.KafkatorioPacketData.Error")
+  data class Error(
+    val message: String? = null,
+    val rawValue: String? = null,
+  ) : KafkatorioPacketData()
+
+}

modules/events-server/build.gradle.kts

@@ -1,12 +1,14 @@
 import dev.adamko.kafkatorio.task.KafkaConsumerGroupsTask
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
+
 plugins {
   dev.adamko.kafkatorio.lang.`kotlin-jvm`
   kotlin("plugin.serialization")
   application
 }
 
+
 description = """
     Receive raw Factorio Events from Kafka and process them into targeted topics or data formats. 
     
@@ -17,6 +19,7 @@ val projectId: String by project.extra
 
 val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
 
+
 dependencies {
   implementation(platform(projects.modules.versionsPlatform))
 
@@ -42,6 +45,11 @@ dependencies {
 
   implementation(libs.okio.core)
 
+  implementation(libs.ktor.network)
+  implementation(libs.ktor.networkTls)
+
+  implementation(libs.simpleSyslogParser)
+
   implementation(libs.kafka.kotkaStreams) {
     isChanging = true
   }
@@ -49,9 +57,6 @@ dependencies {
   implementation(libs.bundles.hoplite)
 }
 
-//configurations.all {
-//  resolutionStrategy.cacheDynamicVersionsFor(1, TimeUnit.MINUTES)
-//}
 
 application {
   mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
@@ -93,7 +98,3 @@ idea {
     excludeDirs = excludeDirs + kafkaStateDir.asFile
   }
 }
-
-//tasks.withType<KotlinNpmInstallTask>().all {
-//  notCompatibleWithConfigurationCache("error: Projects must be configuring")
-//}

modules/events-server/docker/.env

@@ -1,3 +1,3 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
-KAFKATORIO_VERSION=0.6.0
+KAFKATORIO_VERSION=0.6.1

modules/events-server/docker/docker-compose.yml

@@ -15,16 +15,16 @@ services:
     environment:
       KAFKA_BOOTSTRAP_SERVERS: "http://kafka:29092"
       # KAFKA_STREAMS_STATE_DIR: "http://kafka:29092"
-      SERVER_PORT: "12081"
+      SERVER_PORT: "8080"
     ports:
-      - "12080:12081" # http and ws
+      - "12080:8080" # http and ws
     networks:
       - "kafka-nw"
       - "kafkatorio-nw"
       - "traefik-nw"
     labels:
       - "traefik.enable=true"
-      - "traefik.http.services.events-server-kafkatorio.loadbalancer.server.port=12081"
+#      - "traefik.http.services.events-server-kafkatorio.loadbalancer.server.port=12081"
 
 networks:
   kafka-nw:

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt

@@ -1,6 +1,9 @@
 package dev.adamko.kafkatorio.processor
 
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
+import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG
 import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer
 import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer
 import dev.adamko.kafkatorio.processor.topology.broadcastToWebsocket
 import dev.adamko.kafkatorio.processor.topology.colourMapChunks
@@ -17,10 +20,17 @@ import kotlinx.coroutines.CoroutineName
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.cancel
 import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.flow.filter
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.job
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.suspendCancellableCoroutine
+import org.apache.kafka.clients.producer.KafkaProducer
 import org.apache.kafka.clients.producer.ProducerConfig
+import org.apache.kafka.clients.producer.ProducerRecord
+import org.apache.kafka.common.header.internals.RecordHeader
+import org.apache.kafka.common.serialization.Serdes
 import org.apache.kafka.streams.KafkaStreams
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.StreamsConfig
@@ -29,12 +39,14 @@ import org.apache.kafka.streams.TopologyDescription
 
 
 internal class KafkatorioTopology(
-  private val websocketServer: WebsocketServer,
   private val appProps: ApplicationProperties,
+  private val websocketServer: WebsocketServer,
+  private val syslogServer: SyslogSocketServer,
 ) {
 
   private val builderScope = CoroutineScope(CoroutineName("KafkaTopology"))
 
+
   suspend fun start() = builderScope.launch {
     splitPackets()
 
@@ -42,8 +54,11 @@ internal class KafkatorioTopology(
     saveTiles()
 
     websocketBroadcaster()
+
+    syslogProducer()
   }
 
+
   private suspend fun splitPackets() {
     val builder = StreamsBuilder()
     val packets = factorioServerPacketStream(builder)
@@ -54,18 +69,21 @@ internal class KafkatorioTopology(
     launchTopology("splitPackets", topology)
   }
 
+
   private suspend fun groupTilesMapChunks() {
     val builder = StreamsBuilder()
     val topology = colourMapChunks(builder)
     launchTopology("groupTilesMapChunks", topology)
   }
 
+
   private suspend fun saveTiles() {
     val builder = StreamsBuilder()
     val topology = saveMapTiles(builder, appProps.tileDir)
     launchTopology("saveTiles", topology)
   }
 
+
   private suspend fun websocketBroadcaster() {
     val builder = StreamsBuilder()
     broadcastToWebsocket(websocketServer, builder)
@@ -75,11 +93,65 @@ internal class KafkatorioTopology(
     launchTopology(
       "websocketBroadcaster", topology, mapOf(
         StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
-        StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${10.milliseconds.inWholeMilliseconds}"
+        StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
       )
     )
   }
 
+
+  private fun syslogProducer(
+    id: String = "syslog-producer"
+  ) {
+    val props: Properties = appProps.kafkaStreamsConfig.toProperties()
+
+    val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
+    props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
+
+
+    val producer: KafkaProducer<String, String> = KafkaProducer(
+      props,
+      Serdes.String().serializer(),
+      Serdes.String().serializer(),
+    )
+
+    builderScope.coroutineContext.job.invokeOnCompletion {
+      producer.close()
+    }
+
+    syslogServer.messages
+      .filter { it.message?.startsWith("KafkatorioPacket:") == true }
+      .onEach { syslogMsg ->
+
+        val kafkatorioMessage = syslogMsg.message?.substringAfter("KafkatorioPacket:")
+
+        val record = ProducerRecord<String, String>(
+          TOPIC_SRC_SERVER_LOG,
+          "syslog-test",
+          kafkatorioMessage,
+        )
+
+        val headers = SyslogFieldKeys.values()
+          .filter { "HEADER_" in it.name }
+          .mapNotNull { headerKey ->
+            val headerValue = syslogMsg.src[headerKey.field]
+            when {
+              headerValue.isNullOrBlank() -> null
+              else                        -> RecordHeader(
+                headerKey.field,
+                headerValue.toByteArray()
+              )
+            }
+          }
+
+        headers.forEach { header ->
+          record.headers().add(header)
+        }
+
+        producer.send(record)
+      }.launchIn(builderScope)
+  }
+
+
   private suspend fun launchTopology(
     id: String,
     topology: Topology,
@@ -105,7 +177,7 @@ internal class KafkatorioTopology(
         streams.setStateListener { newState: KafkaStreams.State, _ ->
           when (newState) {
             KafkaStreams.State.NOT_RUNNING,
-            KafkaStreams.State.ERROR -> cont.resume(newState)
+            KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
             else                     -> Unit // do nothing
           }
         }
@@ -124,6 +196,7 @@ internal class KafkatorioTopology(
     }
   }
 
+
   private fun printTopologyDescription(appId: String, topology: Topology) {
     val description: TopologyDescription = topology.describe()
 
@@ -143,6 +216,7 @@ internal class KafkatorioTopology(
     )
   }
 
+
   companion object {
     private fun Map<String, String>.toProperties(): Properties =
       entries.fold(Properties()) { props, (k, v) ->

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt

@@ -56,6 +56,7 @@ val KafkatorioPacketData.topicName: String
     is EntityUpdate               -> packetTopicNames.getValue(EntityUpdate::class)
     is MapChunkUpdate             -> packetTopicNames.getValue(MapChunkUpdate::class)
     is PlayerUpdate               -> packetTopicNames.getValue(PlayerUpdate::class)
+    is KafkatorioPacketData.Error -> packetTopicNames.getValue(KafkatorioPacketData.Error::class)
   }
 
 
@@ -68,6 +69,7 @@ private val packetTopicNames: Map<KClass<out KafkatorioPacketData>, String> = ma
   EntityUpdate::class to "entity",
   MapChunkUpdate::class to "map-chunk",
   PlayerUpdate::class to "player",
+  KafkatorioPacketData.Error::class to "error",
 ).mapValues { (_, v) ->
   "$DOMAIN.packet.$v"
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt

@@ -12,6 +12,9 @@ data class ApplicationProperties(
   val tileDir: Path,
   val kafkaAdminConfig: Map<String, String>,
   val kafkaStreamsConfig: Map<String, String>,
+
+  val socketServerHost: String,
+  val socketServerPort: Int,
 ) {
 
   companion object {

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt

@@ -3,6 +3,7 @@ package dev.adamko.kafkatorio.processor
 
 import dev.adamko.kafkatorio.processor.admin.KafkatorioKafkaAdmin
 import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer
 import dev.adamko.kafkatorio.processor.tileserver.WebMapTileServer
 import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer
 import dev.adamko.kafkatorio.processor.tileserver.webServer
@@ -11,7 +12,6 @@ import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.currentCoroutineContext
 import kotlinx.coroutines.job
 import kotlinx.coroutines.launch
-import kotlinx.serialization.Serializable
 import org.http4k.server.Http4kServer
 
 
@@ -22,12 +22,18 @@ suspend fun main() {
   val admin = KafkatorioKafkaAdmin(appProps)
   admin.createKafkatorioTopics()
 
+
+  val syslogServer = SyslogSocketServer(appProps)
   val wsServer = WebsocketServer()
   val tileServer = WebMapTileServer(appProps)
+  val topology = KafkatorioTopology(appProps, wsServer, syslogServer)
 
   coroutineScope {
     launch {
-      val topology = KafkatorioTopology(wsServer, appProps)
+      syslogServer.start()
+    }
+
+    launch {
       topology.start()
 
       println("launched KafkatorioTopology")

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessor.kt

@@ -30,14 +30,17 @@ class DebounceProcessor<K, V>(
   private val storeName: String,
 ) : Processor<K, V, K, V> {
 
+
   private val checkInterval: DurationJvm = checkInterval.toJavaDuration()
 
   private lateinit var state: State<K, V>
 
+
   override fun init(context: ProcessorContext<K, V>) {
     state = Active(context)
   }
 
+
   override fun process(record: Record<K, V>) {
     (state as Active).apply {
       val (key, value, _) = record
@@ -49,21 +52,28 @@ class DebounceProcessor<K, V>(
     }
   }
 
+
   override fun close() {
     (state as? Active)?.schedule?.cancel()
     state = Closed()
   }
 
+
   private sealed interface State<K, V>
 
+
   private class Closed<K, V> : State<K, V>
 
+
   private inner class Active(
     val context: ProcessorContext<K, V>
   ) : State<K, V> {
     val store: TimestampedKeyValueStore<K, V> = context.getStateStore(storeName)
-    val schedule: Cancellable =
+    val schedule: Cancellable = context.schedule(
-      context.schedule(checkInterval, PunctuationType.WALL_CLOCK_TIME, ::checkRecords)
+      checkInterval,
+      PunctuationType.WALL_CLOCK_TIME,
+      ::checkRecords
+    )
 
     private fun checkRecords(now: Long) {
       store.useAll {

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt

@@ -0,0 +1,168 @@
+package dev.adamko.kafkatorio.processor.tileserver
+
+import com.github.palindromicity.syslog.AllowableDeviations
+import com.github.palindromicity.syslog.SyslogParser
+import com.github.palindromicity.syslog.SyslogParserBuilder
+import com.github.palindromicity.syslog.SyslogSpecification
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.processor.tileserver.SyslogMsg.Companion.toSyslogMsg
+import io.ktor.network.selector.SelectorManager
+import io.ktor.network.sockets.aSocket
+import io.ktor.network.sockets.awaitClosed
+import io.ktor.network.sockets.openReadChannel
+import io.ktor.utils.io.readUTF8Line
+import java.util.EnumSet
+import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.SharedFlow
+import kotlinx.coroutines.flow.asSharedFlow
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.yield
+
+
+class SyslogSocketServer(
+  appProps: ApplicationProperties,
+  dispatcher: CoroutineDispatcher = Dispatchers.IO,
+) {
+
+  private val selectorManager: SelectorManager = SelectorManager(dispatcher)
+
+
+  private val serverSocketTcp = aSocket(selectorManager)
+    .tcp()
+    .bind(
+      appProps.socketServerHost,
+      appProps.socketServerPort,
+    )
+
+
+//  private val serverSocketUdp: BoundDatagramSocket = aSocket(selectorManager)
+//    .udp()
+//    .bind(
+//      localAddress = InetSocketAddress(
+//        appProps.socketServerHost,
+//        appProps.socketServerPortUdp,
+//      ),
+//    )
+
+
+  private val syslogParser: SyslogParser = SyslogParserBuilder()
+    .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION))
+    .forSpecification(SyslogSpecification.RFC_5424)
+    .build()
+
+
+  private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
+  val messages: SharedFlow<SyslogMsg>
+    get() = _messages.asSharedFlow()
+
+
+  suspend fun start() {
+//    udp()
+    tcp()
+  }
+
+//  private suspend fun udp() {
+//    println("[SyslogSocketServer] started listening: ${serverSocketUdp.localAddress}")
+//
+//    serverSocketUdp.incoming
+//      .receiveAsFlow()
+//      .map {udpPacket: Datagram ->
+//        val msg = udpPacket.packet.readText()
+//        println("[SyslogSocketServer] received msg: $msg")
+//        syslogParser.parseLine(msg).toSyslogMsg()
+//      }.collect(_messages)
+//  }
+
+  private suspend fun tcp() = coroutineScope {
+    println("[SyslogSocketServer] started listening: ${serverSocketTcp.localAddress}")
+
+    while (isActive) {
+      val socket = serverSocketTcp.accept()
+
+      launch {
+        val channel = socket.openReadChannel()
+        try {
+          while (socket.socketContext.isActive) {
+            val line = channel.readUTF8Line()
+            println("[SyslogSocketServer] received: $line")
+
+            val syslog = syslogParser.parseLine(line).toSyslogMsg()
+
+            _messages.emit(syslog)
+
+            yield()
+          }
+        } catch (e: Exception) {
+          socket.awaitClosed()
+        }
+      }
+    }
+
+    yield()
+  }
+}
+
+
+data class SyslogMsg(
+  val src: Map<String, String?>
+) {
+
+  val message: String?
+    get() = src[SyslogFieldKeys.MESSAGE.field]
+
+  val headerAppName: String?
+    get() = src[SyslogFieldKeys.HEADER_APPNAME.field]
+
+  val headerHostName: String?
+    get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field]
+
+  val headerPri: String?
+    get() = src[SyslogFieldKeys.HEADER_PRI.field]
+
+  val headerPriSeverity: String?
+    get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field]
+
+  val headerPriFacility: String?
+    get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field]
+
+  val headerProcId: String?
+    get() = src[SyslogFieldKeys.HEADER_PROCID.field]
+
+  val headerTimestamp: String?
+    get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field]
+
+  val headerMsgId: String?
+    get() = src[SyslogFieldKeys.HEADER_MSGID.field]
+
+  val headerVersion: String?
+    get() = src[SyslogFieldKeys.HEADER_VERSION.field]
+
+  val structuredBase: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_BASE.field]
+
+  val structuredElementIdFmt: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field]
+
+  val structuredElementIdPnameFmt: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field]
+
+  val structuredElementIdPnamePattern: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field]
+
+  companion object {
+    fun Map<String, Any?>.toSyslogMsg(): SyslogMsg {
+      val map = entries.mapNotNull { (k, v) ->
+        when (v) {
+          is String -> k to v
+          else      -> null
+        }
+      }.toMap()
+      return SyslogMsg(map)
+    }
+  }
+}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt

@@ -17,6 +17,7 @@ import dev.adamko.kotka.extensions.groupedAs
 import dev.adamko.kotka.extensions.materializedAs
 import dev.adamko.kotka.extensions.producedAs
 import dev.adamko.kotka.extensions.repartitionedAs
+import dev.adamko.kotka.extensions.stream
 import dev.adamko.kotka.extensions.streams.filter
 import dev.adamko.kotka.extensions.streams.map
 import dev.adamko.kotka.extensions.streams.mapValues
@@ -55,26 +56,30 @@ fun colourMapChunks(builder: StreamsBuilder): Topology {
 
   chunkTilesColouredTable032.streamMapChunkColouredTo(
     ChunkSize.CHUNK_032,
-    TOPIC_MAP_CHUNK_COLOURED_032_STATE
+    TOPIC_MAP_CHUNK_COLOURED_032_STATE,
   )
 
-  val chunkTilesColoured032Debounced: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
+  val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
     builder.stream(
-      TOPIC_MAP_CHUNK_COLOURED_STATE,
       consumedAs(
-        "$pid.consume.map-chunks-coloured",
+        "$pid.consume.map-chunks-debounced",
-        kxsBinary.serde(),
+        kxsBinary.serde(ServerMapChunkId.serializer()),
-        kxsBinary.serde(),
+        kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
       ),
+      TOPIC_MAP_CHUNK_COLOURED_STATE,
     )
 
   mapOf(
-    ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
+//    ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
-    ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
+//    ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
-    ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
+//    ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
+//    ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
+
+    ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
     ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
+
   ).forEach { (from, to) ->
-    chunkTilesColoured032Debounced
+    chunkTilesColoured
       .filter("$pid.combine-chunks.filter-$from-to-$to") { key, _ ->
         key.chunkSize == from
       }.groupByChunkPosition(to)
@@ -86,9 +91,9 @@ fun colourMapChunks(builder: StreamsBuilder): Topology {
       namePrefix = "groupTilesMapChunks",
       sourceTopic = TOPIC_MAP_CHUNK_COLOURED_032_STATE,
       sinkTopic = TOPIC_MAP_CHUNK_COLOURED_STATE,
-      inactivityDuration = 15.seconds,
+      inactivityDuration = 5.seconds,
-      keySerde = kxsBinary.serde<ServerMapChunkId>(),
+      keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
-      valueSerde = kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
+      valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
     )
 }
 
@@ -182,7 +187,7 @@ private fun KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.enr
     val missingProtos = mutableSetOf<TileProtoHashCode>()
 
     val tileColours = tiles.map.mapValues { (_, code) ->
-      colourDict.map.getOrElse(code) {
+      colourDict.getOrElse(code) {
         missingProtos.add(code)
         ColourHex.TRANSPARENT
       }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt

@@ -5,7 +5,9 @@ import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG
 import dev.adamko.kafkatorio.processor.admin.topicName
 import dev.adamko.kafkatorio.processor.serdes.jsonMapper
 import dev.adamko.kafkatorio.schema.common.FactorioServerId
+import dev.adamko.kafkatorio.schema.common.Tick
 import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
+import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
 import dev.adamko.kotka.extensions.component1
 import dev.adamko.kotka.extensions.component2
 import dev.adamko.kotka.extensions.consumedAs
@@ -31,8 +33,26 @@ fun factorioServerPacketStream(
       Serdes.String(),
     )
   ).map("decode-packets") { serverId: String, value: String ->
-    val packet = jsonMapper.decodeFromString<KafkatorioPacket>(value)
+    val packet = runCatching {
+      jsonMapper.decodeFromString<KafkatorioPacket>(value)
+    }.getOrElse { e ->
+     KafkatorioPacket(
+        modVersion = "unknown",
+        tick = Tick(0u),
+        data = KafkatorioPacketData.Error(
+          message = e.message,
+          rawValue = value,
+        )
+      )
+    }
+
+    if (packet.data is KafkatorioPacketData.Error) {
+      println("error parsing $TOPIC_SRC_SERVER_LOG message: ${packet.data}")
+    }
+
     FactorioServerId(serverId) to packet
+  }.filter { _, value ->
+    value.data !is KafkatorioPacketData.Error
   }
 }
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt

@@ -1 +0,0 @@
-package dev.adamko.kafkatorio.processor.topology

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt

@@ -29,7 +29,9 @@ value class TileProtoHashCode private constructor(val code: Int) {
 
 @JvmInline
 @Serializable
-value class TileColourDict(val map: Map<TileProtoHashCode, ColourHex>)
+value class TileColourDict(
+  private val map: Map<TileProtoHashCode, ColourHex>
+) : Map<TileProtoHashCode, ColourHex> by map
 
 
 /** Get the latest map-tile colours per server. */
@@ -48,9 +50,9 @@ fun KStream<FactorioServerId, PrototypesUpdate>.createTilePrototypeTable()
 
     TileColourDict(map)
   }.filter("$pid.filterMapTileProtos") { _, dict: TileColourDict ->
-    dict.map.isNotEmpty()
+    dict.isNotEmpty()
   }.peek { serverId, dict: TileColourDict ->
-    println("$pid: server $serverId has TileColourDict[${dict.map.size}]: ${dict.map.entries.joinToString()}")
+    println("$pid: server $serverId has TileColourDict[${dict.size}]: ${dict.entries.joinToString()}")
   }.repartition(
     repartitionedAs(
       "$pid.pre-table-repartition",

modules/events-server/src/main/resources/config.yml

@@ -1,12 +1,18 @@
 # Factorio Events Processor config
 
-web-port: ${SERVER_PORT:-12080}
+web-port: ${SERVER_PORT:-8080}
 tile-dir: ${TILE_DIR:-./tile-dir}
 
+
+socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}
+socket-server-port: ${SOCKET_SERVER_PORT:-9002}
+
+
 kafka-admin-config:
   client.id: kafkatorio.events-server-admin
   bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}
 
+
 kafka-streams-config:
   application.id: kafkatorio-events-processor
   bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}

modules/infra-factorio-client/build.gradle.kts

@@ -1,3 +1,4 @@
+import dev.adamko.kafkatorio.factoriomod.FactorioMod
 import dev.adamko.kafkatorio.gradle.asConsumer
 import dev.adamko.kafkatorio.gradle.factorioModAttributes
 import dev.adamko.kafkatorio.gradle.not
@@ -34,10 +35,11 @@ dependencies {
   factorioMod(projects.modules.eventsMod)
 }
 
+
 //<editor-fold desc="Mod deployment tasks">
-val deployModToClient by tasks.registering(Copy::class) {
+val deployModToLocalClient by tasks.registering(Copy::class) {
   description = "Copy the mod to the Factorio client"
-  group = project.name
+  group = FactorioMod.TASK_GROUP
 
   onlyIf { clientModsDirectory.orNull?.asFile?.exists() == true }
 
@@ -50,19 +52,20 @@ val deployModToClient by tasks.registering(Copy::class) {
 }
 //</editor-fold>
 
+
 //<editor-fold desc="Factorio client lifecycle tasks">
 fun ExecOperations.isFactorioRunning(): Spec<Task> = ProcessRunningSpec(this, "factorio.exe")
 
 
 val clientLaunch by tasks.registering(Exec::class) {
-  description = "Run Factorio Steam game client"
+  description = "Run local Factorio Steam game client"
-  group = project.name
+  group = FactorioMod.TASK_GROUP
 
   onlyIf(!serviceOf<ExecOperations>().isFactorioRunning())
   onlyIf { steamExe.orNull?.asFile?.exists() == true }
   onlyIf { clientModsDirectory.orNull?.asFile?.exists() == true }
 
-  dependsOn(deployModToClient)
+  dependsOn(deployModToLocalClient)
   mustRunAfter(clientKill, ":modules:infra-factorio-server:processRun")
 
   commandLine = parseSpaceSeparatedArgs(
@@ -74,8 +77,8 @@ val clientLaunch by tasks.registering(Exec::class) {
 }
 
 val clientKill by tasks.registering(Exec::class) {
-  description = "Run Factorio Steam game client"
+  description = "Stop the local Factorio Steam game client"
-  group = project.name
+  group = FactorioMod.TASK_GROUP
 
   onlyIf(serviceOf<ExecOperations>().isFactorioRunning())
 
@@ -84,11 +87,16 @@ val clientKill by tasks.registering(Exec::class) {
 }
 //</editor-fold>
 
-tasks.build { dependsOn(deployModToClient) }
+
+tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) {
+  group = FactorioMod.TASK_GROUP
+  dependsOn(deployModToLocalClient)
+}
+
 
 tasks.processRun {
   dependsOn(
-    deployModToClient,
+    deployModToLocalClient,
     clientLaunch,
   )
 }

modules/infra-factorio-server/build.gradle.kts

@@ -1,3 +1,4 @@
+import dev.adamko.kafkatorio.factoriomod.FactorioMod
 import dev.adamko.kafkatorio.gradle.asConsumer
 import dev.adamko.kafkatorio.gradle.factorioModAttributes
 import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
@@ -6,21 +7,25 @@ plugins {
   dev.adamko.kafkatorio.infra.`docker-compose`
 }
 
+
 val dockerSrcDir: Directory by extra
 val factorioServerDataDir: Directory = dockerSrcDir.dir("factorio-server")
 
+
 val factorioMod: Configuration by configurations.creating {
   asConsumer()
   factorioModAttributes(objects)
 }
 
+
 dependencies {
   factorioMod(projects.modules.eventsMod)
 }
 
-val deployModToServer by tasks.registering(Copy::class) {
+
+val deployModToLocalServer by tasks.registering(Copy::class) {
   description = "Copy the mod to the Factorio Docker server"
-  group = project.name
+  group = FactorioMod.TASK_GROUP
 
   dependsOn(factorioMod)
 
@@ -34,22 +39,29 @@ val deployModToServer by tasks.registering(Copy::class) {
   }
 }
 
+
 tasks.dockerDown {
   commandLine = parseSpaceSeparatedArgs(""" docker-compose stop """)
 }
 
+
 tasks.dockerUp {
   dependsOn(
-    deployModToServer,
+    deployModToLocalServer,
     ":modules:infra-kafka-cluster:processRun",
   )
 }
 
+
 tasks.dockerEnv {
   properties("FACTORIO_VERSION" to libs.versions.factorio.get())
 }
 
-tasks.build { dependsOn(deployModToServer) }
+
+tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) {
+  group = FactorioMod.TASK_GROUP
+  dependsOn(deployModToLocalServer)
+}
 
 
 idea {

modules/infra-factorio-server/src/.env

@@ -1,4 +1,4 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
 FACTORIO_VERSION=1.1.59
-KAFKATORIO_VERSION=0.6.0
+KAFKATORIO_VERSION=0.6.1

modules/infra-factorio-server/src/docker-compose.yml

@@ -4,8 +4,6 @@ services:
 
   factorio-server:
     image: factoriotools/factorio:${FACTORIO_VERSION}
-    #    logging:
-    #      driver: "json-file"
     container_name: "factorio-server"
     restart: "no"
     networks:
@@ -18,12 +16,11 @@ services:
       - "27015:27015/tcp" # rcon
     volumes:
       - "./factorio-server:/factorio"
-  #    logging:
+    logging:
-  #      driver: "mickyg/kafka-logdriver:latest"
+      driver: syslog
-  #      options:
+      options:
-  #        KAFKA_BROKER_ADDR: "kafka-kraft"
+        syslog-address: "tcp://host.docker.internal:9002"
-  #        LOG_TOPIC: "dockerlogs-factorio-server"
+        syslog-format: "rfc5424"
-  #        LOG_TAG: "factorio-server"
 
   #    environment:
   #      RCON_PORT: 27715

modules/infra-kafka-cluster/src/.env

@@ -1,7 +1,7 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
 GRAFANA_VERSION=8.5.0
-KAFKATORIO_VERSION=0.6.0
+KAFKATORIO_VERSION=0.6.1
 KAFKA_DL_URL=https\://dlcdn.apache.org/kafka/3.2.0/kafka_2.13-3.2.0.tgz
 KAFKA_VERSION=3.2.0
 PROMETHEUS_VERSION=v2.34.0

modules/infra-kafka-cluster/src/docker-compose.yml

@@ -36,8 +36,8 @@ services:
       - "traefik-nw"
     labels:
       - "traefik.enable=true"
-    # ports:
+    ports:
-    #   - "8080:8080"
+      - "18090:8080"
     environment:
       KAFKA_CLUSTERS_0_NAME: "${COMPOSE_PROJECT_NAME}"
       KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka:29092"
@@ -46,20 +46,6 @@ services:
       KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: "http://connect:8083"
 #      - KAFKA_CLUSTERS_0_ZOOKEEPER=kafka:9093
 
-#  kafka-connect:
-#    image: dev-adamko/kafka-connect
-#    container_name: "kafka-connect"
-#    build:
-#      dockerfile: "kafka.Dockerfile"
-#      target: "kafka-connect"
-#      args:
-#        KAFKA_DL_URL: "https://dlcdn.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz"
-#    depends_on:
-#      - kafka-kraft
-#    ports:
-#      - "8083:8083"
-#
-
 #  grafana:
 #    image: "grafana/grafana:${GRAFANA_VERSION}"
 #    container_name: "grafana"

modules/infra-kafka-cluster/src/kafka.Dockerfile

@@ -40,22 +40,3 @@ COPY --chmod=755 ./entrypoint.sh ./entrypoint.sh
 
 # launch the broker in KRaft mode, which means that it runs without ZooKeeper
 ENTRYPOINT ["/kafka/entrypoint.sh"]
-
-
-### Kafka Connect ##
-#FROM openjdk:11 as kafka-connect
-## https://github.com/confluentinc/cp-docker-images/blob/5.3.3-post/debian/kafka-connect-base/Dockerfile
-#
-#ENV COMPONENT=kafka-connect
-#
-#WORKDIR /kafka-connect
-#
-#COPY --from=Kafka-Download /kafka .
-#
-## Default kafka-connect rest.port
-#EXPOSE 8083
-#
-#VOLUME ["/etc/${COMPONENT}/jars", "/etc/${COMPONENT}/secrets"]
-#
-## launch the broker in KRaft mode, which means that it runs without ZooKeeper
-#ENTRYPOINT ["./bin/connect-standalone.sh", "./config/connect-standalone.properties"]

modules/infra-kafka-connect/build.gradle.kts

@@ -1,116 +0,0 @@
-import dev.adamko.kafkatorio.gradle.asConsumer
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
-import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
-
-plugins {
-  dev.adamko.kafkatorio.infra.`docker-compose`
-  dev.adamko.kafkatorio.lang.`kotlin-jvm`
-}
-
-val dockerSrcDir: Directory by extra(layout.projectDirectory.dir("src/main/docker"))
-val connectorJarDir: Directory = dockerSrcDir.dir("camel-kafka-connectors")
-
-val camelConnectorDependencies: Configuration by configurations.creating { asConsumer() }
-
-fun DependencyHandlerScope.kafkaConnector(
-  name: String,
-  group: String = "org.apache.camel.kafkaconnector",
-  version: String = libs.versions.camel.kafkaConnectors.get(),
-  classifier: String = "package",
-  extension: String = "tar.gz",
-) {
-  camelConnectorDependencies(group, name, version, ext = extension, classifier = classifier)
-}
-
-dependencies {
-  implementation(platform(projects.modules.versionsPlatform))
-
-  kotlin("reflect")
-
-  kafkaConnector("camel-ahc-ws-kafka-connector")
-  kafkaConnector("camel-websocket-kafka-connector")
-
-  implementation("com.github.adamko-dev:json5-kotlin:2.0.3")
-
-  implementation(libs.bundles.http4k)
-
-  implementation(libs.kotlinx.serialization.core)
-  implementation(libs.kotlinx.serialization.json)
-}
-
-// disabled for now...
-val projectEnabled = false
-
-val dockerBuildKafkaConnect by tasks.registering(Exec::class) {
-  enabled = projectEnabled
-
-  group = "docker-compose"
-  dependsOn(downloadCamelConnectors, tasks.dockerEnv)
-
-  logging.captureStandardOutput(LogLevel.LIFECYCLE)
-
-  inputs.dir(dockerSrcDir)
-
-  workingDir = dockerSrcDir.asFile
-  commandLine = parseSpaceSeparatedArgs(""" docker-compose build connect """)
-}
-//tasks.assemble { dependsOn(dockerBuildKafkaConnect) }
-
-
-val downloadCamelConnectors by tasks.registering(Sync::class) {
-  enabled = projectEnabled
-
-  group = project.name
-  description = "Retrieve Camel Kafka Connectors from Maven, in preparation for the Docker build"
-
-  dependsOn(camelConnectorDependencies)
-
-  from(
-    provider { camelConnectorDependencies }
-      .map { eventsSchema ->
-        eventsSchema.incoming
-          .artifacts
-          .artifactFiles
-      }
-  )
-  into(temporaryDir)
-
-  doLast {
-    sync {
-      fileTree(temporaryDir)
-        .asFileTree
-        .forEach { zip ->
-          from(tarTree(zip))
-        }
-      into(connectorJarDir)
-    }
-  }
-}
-
-tasks.dockerUp {
-  enabled = projectEnabled
-
-  dependsOn(":modules:infra-kafka-cluster:dockerUp")
-  dependsOn(downloadCamelConnectors)
-}
-
-tasks.dockerEnv {
-  enabled = projectEnabled
-
-  properties(
-    "CONNECTOR_ID" to "kafkatorio-connect",
-    "KAFKA_CONNECT_VERSION" to libs.versions.confluent.kafkaConnect.get(),
-  )
-}
-
-idea {
-  module {
-    excludeDirs.add(file("src/main/docker/camel-kafka-connectors"))
-  }
-}
-
-tasks.withType<KotlinCompile>().configureEach {
-  kotlinOptions.freeCompilerArgs += listOf(
-    "-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
-  )
-}

modules/infra-kafka-connect/src/main/docker/.env

@@ -1,4 +0,0 @@
-# Do not edit manually. This file was created with task 'dockerEnv'
-COMPOSE_PROJECT_NAME=kafkatorio
-CONNECTOR_ID=kafkatorio-connect
-KAFKA_CONNECT_VERSION=7.0.1

modules/infra-kafka-connect/src/main/docker/.gitignore

@@ -1 +0,0 @@
-camel-kafka-connectors/

modules/infra-kafka-connect/src/main/docker/docker-compose.yml

@@ -1,55 +0,0 @@
-version: "3.9"
-
-services:
-
-  connect:
-    image: adamko-dev/kafkatorio-kafka-connect:${KAFKA_CONNECT_VERSION}
-    hostname: ${CONNECTOR_ID}
-    container_name: ${CONNECTOR_ID}
-    build:
-      context: "."
-      dockerfile: kafka-connect.Dockerfile
-      args:
-        KAFKA_CONNECT_VERSION: ${KAFKA_CONNECT_VERSION}
-#        FILE_PULSE_VERSION: ${CONNECTOR_FILE_PULSE_VERSION}
-#        CAMEL_KC_VERSION: ${CONNECTOR_CAMEL_KAFKA_VERSION}
-#        CAMEL_CONNECTOR: ${CAMEL_FILE_CONNECTOR}
-    ports:
-      - "8083:8083"
-      - "8095:8095" # ws-async (client?)
-      - "9292:9292" # websocket server(?)
-#    volumes:
-#      - "./connect/input:/warehouse/input"
-#      - "./connect/components:/usr/share/confluent-hub-components"
-
-#    volumes:
-#      - "./camel-kafka-connectors:/usr/share/java"
-    networks:
-      - "kafkatorio-nw"
-    environment:
-      CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
-#      CONNECT_REST_ADVERTISED_HOST_NAME: ${CONNECTOR_ID}
-      CONNECT_REST_ADVERTISED_HOST_NAME: connect
-#      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://${SCHEMA_REGISTRY}:8081"
-#      CONNECT_REST_PORT: 28083
-      CONNECT_REST_PORT: 8083
-      CONNECT_GROUP_ID: ${CONNECTOR_ID}
-      CONNECT_CONFIG_STORAGE_TOPIC: ${CONNECTOR_ID}-configs
-      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
-      CONNECT_OFFSET_STORAGE_TOPIC: ${CONNECTOR_ID}-offsets
-      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_STATUS_STORAGE_TOPIC: ${CONNECTOR_ID}-status
-      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
-#      CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
-      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
-      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/camel-kafka-connectors"
-      CONNECT_LOG4J_ROOT_LOGLEVEL: INFO
-      CONNECT_LOG4J_LOGGERS: "org.apache.zookeeper=INFO,org.I0Itec.zkclient=ERROR,org.reflections=ERROR"
-#      CONNECT_LISTENERS: "http://localhost:28083"
-
-networks:
-  kafkatorio-nw:
-    name: "kafkatorio-nw"
-    external: true

modules/infra-kafka-connect/src/main/docker/kafka-connect.Dockerfile

@@ -1,19 +0,0 @@
-ARG KAFKA_CONNECT_VERSION
-
-
-FROM confluentinc/cp-kafka-connect-base:${KAFKA_CONNECT_VERSION}
-
-ENV CONNECT_VERSION=${KAFKA_CONNECT_VERSION}
-
-USER 0:0
-
-COPY ./camel-kafka-connectors /usr/share/camel-kafka-connectors
-
-#WORKDIR /usr/share/java/
-#
-##ENV CAMEL_KC_VERSION=0.9.0
-##ENV CAMEL_CONNECTOR=camel-file-kafka-connector
-#
-#ENV URL=https://repo.maven.apache.org/maven2/org/apache/camel/kafkaconnector/camel-file-kafka-connector/0.9.0/camel-file-kafka-connector-0.9.0-package.tar.gz
-#
-#RUN curl -L $URL | tar -xzvf - --strip=0 $CAMEL_CONNECTOR

modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/Resources.kt

@@ -1,10 +0,0 @@
-package dev.adamko.kafkatorio.kafkaconnect
-
-import java.io.File
-
-object Resources {
-  fun loadResource(name: String): File =
-    File(requireNotNull(Resources::class.java.getResource(name)?.file) {
-      "could not load resource '$name'"
-    })
-}

modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt

@@ -1,92 +0,0 @@
-package dev.adamko.kafkatorio.kafkaconnect
-
-import at.syntaxerror.json5.Json5Module
-import kotlinx.serialization.decodeFromString
-import kotlinx.serialization.encodeToString
-import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.JsonElement
-import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.jsonPrimitive
-import org.http4k.client.OkHttp
-import org.http4k.core.Method.GET
-import org.http4k.core.Method.PUT
-import org.http4k.core.Request
-import org.http4k.core.Uri
-
-val jsonMapper = Json {
-  prettyPrint = true
-  prettyPrintIndent = "  "
-}
-
-val j5 = Json5Module()
-
-fun main() {
-
-  val connectorHostUrl = Uri.of("http://localhost:8083")
-
-  val configFile = Resources.loadResource("/configs/ckc-websocket-lua-objects.json5")
-
-  val jsonObject: JsonObject = j5.decodeObject(configFile.readText())
-  println("json5 connector config: $jsonObject")
-  val configJson = jsonMapper.encodeToString(jsonObject)
-  println("json connector config: $configJson")
-
-  val connectorName = jsonObject["name"]!!.jsonPrimitive.content
-
-  val request: Request =
-    Request(PUT, connectorHostUrl.path("/connectors/$connectorName/config"))
-      .header("Content-Type", "application/json")
-      .header("Accept", "application/json")
-//      .header("Accept-Encoding", "identity")
-      .body(configJson)
-
-  val client = OkHttp()
-
-  // check the status of KC
-  val statusResponse = client(Request(GET, connectorHostUrl))
-  println(statusResponse.toMessage())
-
-  val response = client(request)
-
-  println(response.toMessage())
-  println(
-    jsonMapper.encodeToString(
-      jsonMapper.decodeFromString<JsonElement>(response.body.toString())
-    )
-  )
-}
-
-//val updateConnectors by tasks.registering {
-//  group = project.name
-//
-//  val configFile = layout.projectDirectory.file("configs/websocket-async-lua-objects.json5")
-//  inputs.file(configFile)
-//
-//  doLast {
-//    val fileText = resources.text.fromFile(configFile)
-//
-//    val jsonObject: JsonObject = j5.decodeObject(fileText.asString())
-//
-//    logger.lifecycle("out: $jsonObject")
-//    val configJson = jsonMapper.encodeToString(jsonObject)
-//    logger.lifecycle("configJson: $configJson")
-//
-//    val request = Request(
-//      PUT,
-//      Uri.of("http://localhost:8083").path("/connectors/kafkatorio-ws-connector/config")
-//    ).header("Content-Type", "application/json")
-//      .body(configJson)
-//
-//
-//    val client = OkHttp()
-//    val response = client(request)
-//
-//    println(response.toMessage())
-//    println(
-//      jsonMapper.encodeToString(
-//        jsonMapper.decodeFromString<JsonElement>(response.body.toString())
-//      )
-//    )
-//  }
-//
-//}

modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt

@@ -1,41 +0,0 @@
-package dev.adamko.kafkatorio.kafkaconnect
-
-import java.time.Duration
-import org.http4k.client.WebsocketClient
-import org.http4k.core.Uri
-import org.http4k.websocket.WsMessage
-
-fun main() {
-
-  val client = WebsocketClient.blocking(
-    uri = Uri.of("ws://localhost:9073/ws"),
-//    autoReconnection = true,
-//    timeout = Duration.ofSeconds(30)
-  )
-//  client.send(WsMessage("server sent on connection"))
-
-  client.received()
-    .take(100)
-    .forEachIndexed { index, wsMessage ->
-      println("[$index] ${wsMessage.bodyString()}")
-    }
-
-
-//  val nonBlockingClient = WebsocketClient.nonBlocking(
-//    uri = Uri.of("ws://localhost:9097/lua-objects"),
-//    onError = { it.printStackTrace() },
-//    onConnect = { ws ->
-//      println("onConnect, ws.upgradeRequest ${ws.upgradeRequest}")
-////      ws.send(WsMessage("client sent on connection"))
-//    }
-//  )
-//
-//  nonBlockingClient.onMessage { msg -> print(msg.bodyString()) }
-//  nonBlockingClient.onClose {
-//    println("non-blocking client closing")
-//  }
-////  nonBlockingClient.onError { it.printStackTrace() }
-//
-//  Thread.sleep(Duration.ofMinutes(1).toMillis())
-
-}

modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5

@@ -1,130 +0,0 @@
-{
-  "name": "ckc-websocket-lua-objects",
-  "connector.class": "org.apache.camel.kafkaconnector.websocket.CamelWebsocketSinkConnector",
-  "topics.regex": "^Lua.*$",
-  "key.converter": "org.apache.kafka.connect.storage.StringConverter",
-  "value.converter": "org.apache.kafka.connect.storage.StringConverter",
-  // The hostname. The default value is 0.0.0.0. Setting this option on the component will use the
-  // component configured value as default.
-    "camel.sink.path.host": "0.0.0.0",
-//    "camel.sink.path.host": "host.docker.internal",
-  // The port number. The default value is 9292. Setting this option on the component will use the
-  // component configured value as default.
-  "camel.sink.path.port": "9292",
-  // Name of the websocket channel to use
-  "camel.sink.path.resourceUri": "lua-objects",
-  "camel.sink.endpoint.sendToAll": "true",
-  "camel.component.websocket.host":"0.0.0.0",
-  "camel.component.websocket.port": "8084",
-  // Can be used to set the size in bytes that the websocket created by the websocketServlet may
-  // be accept before closing. (Default is -1 - or unlimited)
-  //    "camel.sink.endpoint.maxBinaryMessageSize": -1,
-
-  // Whether the producer should be started lazy (on the first message). By starting lazy you can
-  // use this to allow CamelContext and routes to startup in situations where a producer may otherwise
-  // fail during starting and cause the route to fail being started. By deferring this startup to be
-  // lazy then the startup failure can be handled during routing messages via Camel's routing error handlers.
-  // Beware that when the first message is processed then creating and starting the producer may take a little
-  // time and prolong the total processing time of the processing.
-  //    "camel.sink.endpoint.lazyStartProducer": false,
-
-  // Timeout in millis when sending to a websocket channel. The default timeout is 30000 (30 seconds).
-  //    "camel.sink.endpoint.sendTimeout": 30000,
-
-  // To send to all websocket subscribers. Can be used to configure on endpoint level, instead of
-  // having to use the WebsocketConstants.SEND_TO_ALL header on the message.
-  //    "camel.sink.endpoint.sendToAll": ,
-
-  // Set the buffer size of the websocketServlet, which is also the max frame byte size (default
-  // 8192)
-  //    "camel.sink.endpoint.bufferSize": 8192,
-
-  // Set the time in ms that the websocket created by the websocketServlet may be idle before
-  // closing. (default is 300000)
-  //    "camel.sink.endpoint.maxIdleTime": 300000,
-
-  // Can be used to set the size in characters that the websocket created by the websocketServlet
-  // may be accept before closing.
-  //    "camel.sink.endpoint.maxTextMessageSize": ,
-
-  // Can be used to set the minimum protocol version accepted for the websocketServlet.
-  // (Default 13 - the RFC6455 version)
-  //    "camel.sink.endpoint.minVersion": 13,
-
-  // The CORS allowed origins. Use to allow all.
-  //    "camel.sink.endpoint.allowedOrigins": ,
-
-  // Whether to enable CORS
-  //    "camel.sink.endpoint.crossOriginFilterOn": false,
-
-  // Context path for filtering CORS
-  //    "camel.sink.endpoint.filterPath": ,
-
-  // If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX
-  // support for more details.
-  //    "camel.sink.endpoint.enableJmx": false,
-
-  // To configure security using SSLContextParameters
-  //    "camel.sink.endpoint.sslContextParameters": ,
-
-  // The hostname. The default value is 0.0.0.0
-  //    "camel.component.websocket.host": "0.0.0.0",
-
-  // The port number. The default value is 9292
-  //    "camel.component.websocket.port": 9292,
-
-  // Whether the producer should be started lazy (on the first message). By starting lazy you can
-  // use this to allow CamelContext and routes to startup in situations where a producer may otherwise
-  // fail during starting and cause the route to fail being started. By deferring this startup to be
-  // lazy then the startup failure can be handled during routing messages via Camel's routing error handlers.
-  // Beware that when the first message is processed then creating and starting the producer may take a little
-  // time and prolong the total processing time of the processing.
-  //    "camel.component.websocket.lazyStartProducer": false,
-
-  // Whether autowiring is enabled. This is used for automatic autowiring options (the option must
-  // be marked as autowired) by looking up in the registry to find if there is a single instance
-  // of matching type, which then gets configured on the component. This can be used for automatic
-  // configuring JDBC data sources, JMS connection factories, AWS Clients, etc.
-  //    "camel.component.websocket.autowiredEnabled": true,
-
-  // If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX
-  // support for more details.
-  //    "camel.component.websocket.enableJmx": false,
-
-  // To set a value for maximum number of threads in server thread pool. MaxThreads\/minThreads
-  // or threadPool fields are required due to switch to Jetty9. The default values for maxThreads
-  // is 1 2 noCores.
-  //    "camel.component.websocket.maxThreads": ,
-
-  // To set a value for minimum number of threads in server thread pool. MaxThreads\/minThreads
-  // or threadPool fields are required due to switch to Jetty9. The default values for minThreads
-  // is 1.
-  //    "camel.component.websocket.minThreads": ,
-
-  // This is a comma-separated list of subprotocols that are supported by the application. The list
-  // is in priority order. The first subprotocol on this list that is proposed by the client is
-  // the one that will be accepted. If no subprotocol on this list is proposed by the client, then the
-  // websocket connection is refused. The special value 'any' means that any subprotocol is acceptable. 'any'
-  // can be used on its own, or as a failsafe at the end of a list of more specific protocols. 'any' will also
-  // match the case where no subprotocol is proposed by the client.
-  //    "camel.component.websocket.subprotocol": "any",
-
-  // To use a custom thread pool for the server. MaxThreads\/minThreads or threadPool fields are
-  // required due to switch to Jetty9.
-  //    "camel.component.websocket.threadPool": ,
-
-  // To configure security using SSLContextParameters
-  //    "camel.component.websocket.sslContextParameters": ,
-
-  // The password for the keystore when using SSL.
-  //    "camel.component.websocket.sslKeyPassword": ,
-
-  // The path to the keystore.
-  //    "camel.component.websocket.sslKeystore": ,
-
-  // The password when using SSL.
-  //    "camel.component.websocket.sslPassword": ,
-
-  // Enable usage of global SSL context parameters.
-  //    "camel.component.websocket.useGlobalSslContextParameters": false,
-}

modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5

@@ -1,84 +0,0 @@
-//Exchange data with external Websocket servers using Async Http Client
-{
-  name: "websocket-async-lua-objects",
-  "connector.class": "org.apache.camel.kafkaconnector.ahcws.CamelAhcwsSinkConnector",
-  "tasks.max": 1,
-  //use the kafka converters that better suit your needs, these are just defaults:
-  "key.converter": "org.apache.kafka.connect.storage.StringConverter",
-  "value.converter": "org.apache.kafka.connect.storage.StringConverter",
-  //https://docs.confluent.io/platform/current/installation/configuration/connect/sink-connect-configs.html#sinkconnectorconfigs_topics
-  //List of topics to consume, separated by commas
-  // topics=
-  "topics.regex": "^Lua.*$",
-
-  // mandatory properties (for a complete properties list see the connector documentation):
-
-  // The URI to use such as http://hostname:port/path
-  "camel.sink.path.httpUri": "host.docker.internal:8095/ws/lua",
-  // If the option is true, then the Exchange.HTTP_URI header is ignored, and use the endpoint's
-  // URI for request. You may also set the throwExceptionOnFailure to be false to let the
-  // AhcProducer send all the fault response back.
-  // camel.sink.endpoint.bridgeEndpoint=
-
-  // The initial in-memory buffer size used when transferring data between Camel and AHC Client.
-  // camel.sink.endpoint.bufferSize=
-  // To use a custom HeaderFilterStrategy to filter header to and from Camel message.
-  // camel.sink.endpoint.headerFilterStrategy=
-  // Option to disable throwing the AhcOperationFailedException in case of failed responses from
-  // the remote server. This allows you to get all responses regardless of the HTTP status code.
-  // camel.sink.endpoint.throwExceptionOnFailure=
-  // If enabled and an Exchange failed processing on the consumer side, and if the caused
-
-  // Exception was send back serialized in the response as a application\/x-java-serialized-object
-  // content type (for example using Jetty or Servlet Camel components). On the producer side the
-  // exception will be deserialized and thrown as is, instead of the AhcOperationFailedException.
-  // The caused exception is required to be serialized. This is by default turned off.
-  // If you enable this then be aware that Java will deserialize the incoming data from the
-  // request to Java and that can be a potential security risk.
-  // camel.sink.endpoint.transferException=
-
-  // Define if the Connection Close header has to be added to HTTP Request. This parameter is false by default
-  // camel.sink.endpoint.connectionClose=
-  // Configure a cookie handler to maintain a HTTP session
-  // camel.sink.endpoint.cookieHandler=
-
-  // Whether the producer should be started lazy (on the first message). By starting lazy you can
-  // use this to allow CamelContext and routes to startup in situations where a producer
-  // may otherwise fail during starting and cause the route to fail being started. By deferring
-  // this startup to be lazy then the startup failure can be handled during routing messages via
-  // Camel's routing error handlers. Beware that when the first message is processed then creating
-  // and starting the producer may take a little time and prolong the total processing time of the
-  // processing.
-  // camel.sink.endpoint.lazyStartProducer=
-
-  // To enable streaming to send data as multiple text fragments.
-  // camel.sink.endpoint.useStreaming=
-  // To use a custom AhcBinding which allows to control how to bind between AHC and Camel.
-  // camel.sink.endpoint.binding=
-  // To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance.
-  // camel.sink.endpoint.clientConfig=
-  // To configure the AsyncHttpClientConfig using the key\/values from the Map.
-  // camel.sink.endpoint.clientConfigOptions=
-  // To configure the AsyncHttpClientConfig Realm using the key\/values from the Map.
-  // camel.sink.endpoint.clientConfigRealmOptions=
-  // Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. This reference overrides any configured SSLContextParameters at the component level. See Using the JSSE Configuration Utility. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level.
-  // camel.sink.endpoint.sslContextParameters=
-  // Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail during starting and cause the route to fail being started. By deferring this startup to be lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. Beware that when the first message is processed then creating and starting the producer may take a little time and prolong the total processing time of the processing.
-  // camel.component.ahc-ws.lazyStartProducer=
-  // Whether to allow java serialization when a request uses context-type=application\/x-java-serialized-object This is by default turned off. If you enable this then be aware that Java will deserialize the incoming data from the request to Java and that can be a potential security risk.
-  // camel.component.ahc-ws.allowJavaSerializedObject=
-  // Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching type, which then gets configured on the component. This can be used for automatic configuring JDBC data sources, JMS connection factories, AWS Clients, etc.
-  // camel.component.ahc-ws.autowiredEnabled=
-  // To use a custom AhcBinding which allows to control how to bind between AHC and Camel.
-  // camel.component.ahc-ws.binding=
-  // To use a custom AsyncHttpClient
-  // camel.component.ahc-ws.client=
-  // To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance.
-  // camel.component.ahc-ws.clientConfig=
-  // To use a custom org.apache.camel.spi.HeaderFilterStrategy to filter header to and from Camel message.
-  // camel.component.ahc-ws.headerFilterStrategy=
-  // Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level.
-  // camel.component.ahc-ws.sslContextParameters=
-  // Enable usage of global SSL context parameters.
-  // camel.component.ahc-ws.useGlobalSslContextParameters=
-}

modules/infra-kafka-pipe/src/.env

@@ -1,3 +1,3 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
-KAFKATORIO_VERSION=0.6.0
+KAFKATORIO_VERSION=0.6.1

modules/infra-kafka-pipe/src/docker-compose.yml

@@ -20,10 +20,10 @@ services:
       KAFKA_HOST: "kafka:29092"
     volumes:
       # Windows (requires privileged/root)
-#      - "//var/run/docker.sock:/var/run/docker.sock"
+      - "//var/run/docker.sock:/var/run/docker.sock"
 
       # Rootless:
-      - "/run/user/1000/docker.sock:/var/run/docker.sock"
+#      - "/run/user/1000/docker.sock:/var/run/docker.sock"
 
       # Unix?
       # - "/var/run/docker.sock:/var/run/docker.sock"

modules/web-map/build.gradle.kts

@@ -1,4 +1,3 @@
-import dev.adamko.kafkatorio.gradle.taskProvider
 import org.jetbrains.kotlin.gradle.plugin.KotlinDependencyHandler
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
 import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackConfig
@@ -17,6 +16,7 @@ kvision {
   enableHiddenKotlinJsStore.set(false)
 }
 
+
 rootProject.extensions.configure<NodeJsRootExtension> {
   // https://github.com/rjaros/kvision/issues/410
   versions.webpackCli.version = "4.10.0"
@@ -48,13 +48,9 @@ afterEvaluate {
 
 kotlin {
   js(IR) {
-//    browser()
-//    binaries.executable()
-//  }
-//
-//  js("frontend", IR) {
-//    nodejs()
     browser {
+      binaries.executable()
+
       runTask {
         outputFileName = "main.bundle.js"
         sourceMaps = false
@@ -87,16 +83,11 @@ kotlin {
         }
       }
     }
-    binaries.executable()
 //    compilations["main"].packageJson {
 //      this.devDependencies["webpack-dev-server"] = "4.9.1"
 //    }
   }
 
-//  jvm("backend") {
-//
-//  }
-
   sourceSets {
 
     all {
@@ -106,7 +97,6 @@ kotlin {
       }
     }
 
-//    val commonMain by getting {
     main {
       dependencies {
         implementation(dependencies.platform(libs.kotlinx.coroutines.bom))
@@ -117,13 +107,6 @@ kotlin {
         implementation(projects.modules.eventsSchema)
 
         implementation(libs.kotlinx.html)
-//      }
-//    }
-
-//    val jsMain by getting {
-//      dependsOn(commonMain)
-//
-//      dependencies {
 
         implementation(projects.modules.eventsSchema)
 
@@ -154,37 +137,15 @@ kotlin {
 //        implementation(dependencies.platform(devNpm("webpack-dev-server", "4.9.1")))
       }
 
-//      val webDir = file("src/main/web")
       resources.srcDir("src/main/web")
-//      resources.srcDir("src/main/resources")
-
-//      kotlin.srcDir("src/main/kotlin")
     }
 
-//    val frontendMain by getting {
-//      dependsOn(jsMain)
-////      resources.srcDir("src/main/web")
-////      resources.srcDir("src/main/resources")
-//    }
-//
-//    val jsTest by getting {
-//      dependencies {
-//        implementation(kotlin("test-js"))
-//
-//        kvision("kvision-testutils")
-//
-////        implementation(npm("karma", "^6.3.16"))
-//      }
-//
-//      kotlin.srcDir("src/test/kotlin")
-//    }
 
     test {
       dependencies {
         implementation(kotlin("test-js"))
 
         kvision("kvision-testutils")
-
         //        implementation(npm("karma", "^6.3.16"))
       }
     }
@@ -221,22 +182,22 @@ fun KotlinDependencyHandler.kvision(
 //evaluationDependsOn(projects.modules.eventsSchema.dependencyProject.path)
 
 
-// stop warnings when building
+//// stop warnings when building
-tasks.configureEach {
+//tasks.configureEach {
-  when (name) {
+//  when (name) {
-    "productionExecutableCompileSync"          ->
+//    "productionExecutableCompileSync"          ->
-      dependsOn(taskProvider("compileProductionExecutableKotlinJs"))
+//      dependsOn(taskProvider("compileProductionExecutableKotlinJs"))
-    "testTestDevelopmentExecutableCompileSync" ->
+//    "testTestDevelopmentExecutableCompileSync" ->
-      dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinJs"))
+//      dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinJs"))
-//    "jsTestTestDevelopmentExecutableCompileSync"       ->
+////    "jsTestTestDevelopmentExecutableCompileSync"       ->
-//      dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinFrontend"))
+////      dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinFrontend"))
-//    "jsProductionExecutableCompileSync"                ->
+////    "jsProductionExecutableCompileSync"                ->
-//      dependsOn(taskProvider("compileProductionExecutableKotlinFrontend"))
+////      dependsOn(taskProvider("compileProductionExecutableKotlinFrontend"))
-    "browserDevelopmentRun"                    ->
+//    "browserDevelopmentRun"                    ->
-      dependsOn(taskProvider("browserDevelopmentWebpack"))
+//      dependsOn(taskProvider("browserDevelopmentWebpack"))
-    "browserDevelopmentWebpack"                ->
+//    "browserDevelopmentWebpack"                ->
-      dependsOn(taskProvider("assemble"))
+//      dependsOn(taskProvider("assemble"))
-    "browserProductionWebpack"                 ->
+//    "browserProductionWebpack"                 ->
-      dependsOn(taskProvider("developmentExecutableCompileSync"))
+//      dependsOn(taskProvider("developmentExecutableCompileSync"))
-  }
+//  }
-}
+//}

modules/web-map/docker/.env

@@ -1,3 +1,3 @@
 # Do not edit manually. This file was created with task 'dockerEnv'
 COMPOSE_PROJECT_NAME=kafkatorio
-KAFKATORIO_VERSION=0.6.0
+KAFKATORIO_VERSION=0.6.1

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt

@@ -125,7 +125,7 @@ class FactorioMap(
       minZoom = -2
       maxZoom = 6
 
-      maxNativeZoom = -1
+      maxNativeZoom = 0
       minNativeZoom = -1
 
       noWrap = true

modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt

@@ -5,6 +5,7 @@ import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate
 import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate
 import dev.adamko.kafkatorio.schema.packets.EntityUpdate
 import dev.adamko.kafkatorio.schema.packets.EventServerPacket
+import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
 import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate
 import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
 import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
@@ -83,7 +84,8 @@ class WebsocketService(
               is PrototypesUpdate,
               is SurfaceUpdate,
               is EntityUpdate,
-              is MapChunkUpdate -> {
+              is MapChunkUpdate,
+              is KafkatorioPacketData.Error -> {
                 // to be continued...
               }
             }

settings.gradle.kts

@@ -10,7 +10,6 @@ include(
 
   ":modules:infra-kafka-cluster",
   ":modules:infra-kafka-pipe",
-  ":modules:infra-kafka-connect",
 
   ":modules:kt-rcon",
   ":modules:web-map",
@@ -26,14 +25,5 @@ apply(from = "./buildSrc/repositories.settings.gradle.kts")
 
 @Suppress("UnstableApiUsage") // Central declaration of repositories is an incubating feature
 dependencyResolutionManagement {
-
   repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS)
-
-//  repositories {
-//    maven("https://s01.oss.sonatype.org/content/repositories/snapshots/") {
-////      content {
-////        includeGroup("dev.adamko.kxstsgen")
-////      }
-//    }
-//  }
 }