zoom zoom faster - make 3 different Kafka consumers for player/tile/packet updates

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt

@@ -7,29 +7,38 @@ import dev.adamko.kafkatorio.processor.topology.saveMapTiles
 import dev.adamko.kafkatorio.processor.topology.splitFactorioServerPacketStream
 import dev.adamko.kafkatorio.processor.topology.tileProtoColourDictionary
 import java.time.Duration
+import kotlin.coroutines.CoroutineContext
+import kotlinx.coroutines.CoroutineName
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.job
+import kotlinx.coroutines.launch
 import org.apache.kafka.streams.KafkaStreams
 import org.apache.kafka.streams.StreamsBuilder
-import org.apache.kafka.streams.TopologyDescription
+import org.apache.kafka.streams.StreamsConfig
+import org.apache.kafka.streams.Topology
 
 class KafkatorioTopology(
   private val websocketServer: WebsocketServer,
-  private val builder: StreamsBuilder = StreamsBuilder(),
+) : CoroutineScope {
-) {
 
+  override val coroutineContext: CoroutineContext =
+    Dispatchers.Default + SupervisorJob() + CoroutineName("KafkatorioTopology")
 
   companion object {
     const val sourceTopic = "factorio-server-log"
   }
 
+  fun start() {
+    saveTiles()
+    splitPackets()
+    playerUpdates()
+  }
 
-  fun build() {
+  private fun saveTiles() {
-
+    val builder = StreamsBuilder()
     val packets = factorioServerPacketStream(builder)
-
-    splitFactorioServerPacketStream(packets)
-
-    playerUpdatesToWsServer(websocketServer, builder)
-
     val tileProtoColourDict = tileProtoColourDictionary(packets)
 
     val tilePrototypesTable = groupTilesIntoChunksWithColours(
@@ -40,17 +49,46 @@ class KafkatorioTopology(
 
     val topology = builder.build()
 
-    val streams = KafkaStreams(topology, appProps.kafkaConfig)
+    launchTopology("saveTiles", topology)
+  }
 
-//    streams.cleanUp()
+  private fun splitPackets() {
+
+    val builder = StreamsBuilder()
+    val packets = factorioServerPacketStream(builder)
+    splitFactorioServerPacketStream(packets)
+
+    val topology = builder.build()
+
+    launchTopology("splitPackets", topology)
+  }
+
+  private fun playerUpdates() {
+    val builder = StreamsBuilder()
+    playerUpdatesToWsServer(websocketServer, builder)
+
+    val topology = builder.build()
+
+    launchTopology("playerUpdates", topology)
+  }
+
+  private fun launchTopology(id: String, topology: Topology) {
+
+    val props =  appProps.kafkaConfig
+
+    val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as? String
+    props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
+
+
+    val streams = KafkaStreams(topology, props)
     streams.setUncaughtExceptionHandler(StreamsExceptionHandler())
-    Runtime.getRuntime().addShutdownHook(Thread { streams.close(Duration.ofSeconds(1)) })
 
-    val description: TopologyDescription = topology.describe()
+    coroutineContext.job.invokeOnCompletion {
-    println(description)
+      println("closing $id KafkaStreams")
-
+      streams.close(Duration.ofSeconds(1))
-    streams.start()
+    }
 
+    launch { streams.start() }
   }
 
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt

@@ -45,10 +45,10 @@ class ApplicationProperties(
   val kafkaConfig: Properties
     get() = EnvironmentKey
       .nonEmptyString()
-      .map {
+      .map { srcString ->
-        it.reader().use { sr ->
+        srcString.reader().use { stringReader ->
-          Properties().also { p ->
+          Properties().also { properties ->
-            p.load(sr)
+            properties.load(stringReader)
           }
         }
       }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt

@@ -21,7 +21,7 @@ fun main(): Unit = runBlocking {
   val tileServer = WebMapTileServer()
 
   val topology = KafkatorioTopology(wsServer)
-  topology.build()
+  topology.start()
 
   val webServer = PolyHandler(
     tileServer.build(),