- start migrating tile-server to use Ktor

- start splitting up Kafka processor and tile-server to two subprojects

build.gradle.kts

@@ -7,7 +7,7 @@ plugins {
   `project-report`
   `build-dashboard`
 
-//  kotlin("multiplatform") apply false // try to stop error 'Projects must be configuring'
+  kotlin("multiplatform") apply false // try to stop error 'Projects must be configuring'
 //  kotlin("js") apply false // try to stop error 'Projects must be configuring'
 }
 

buildSrc/build.gradle.kts

@@ -43,14 +43,14 @@ dependencies {
   implementation(libs.kotlinx.cli)
 
   implementation(platform(libs.ktor.bom))
-  implementation(libs.ktor.clientAuth)
+  implementation(libs.ktorClient.auth)
-  implementation(libs.ktor.clientContentNegotiation)
+  implementation(libs.ktorClient.contentNegotiation)
-  implementation(libs.ktor.clientCore)
+  implementation(libs.ktorClient.core)
-  implementation(libs.ktor.clientEncoding)
+  implementation(libs.ktorClient.encoding)
-  implementation(libs.ktor.clientLogging)
+  implementation(libs.ktorClient.logging)
-  implementation(libs.ktor.clientCIO)
+  implementation(libs.ktorClient.cio)
-  implementation(libs.ktor.clientResources)
+  implementation(libs.ktorClient.resources)
-  implementation(libs.ktor.serializationKotlinxJson)
+  implementation(libs.ktorSerialization.kotlinxJson)
 
   // https://github.com/avast/gradle-docker-compose-plugin
 //  implementation("com.avast.gradle:gradle-docker-compose-plugin:${Versions.gradleDockerComposePlugin}")

gradle/libs.versions.toml

@@ -31,7 +31,7 @@ kotlinx-coroutines    = "1.6.1"
 kotlinx-cli           = "0.3.4"
 kotlinx-datetime      = "0.3.3"
 
-ktor                  = "2.0.2"
+ktor                  = "2.0.3"
 
 hoplite               = "2.1.5"
 
@@ -123,45 +123,50 @@ http4k-graphql        = { group = "org.http4k",  name = "http4k-graphql"
 ## ktor ##
 ktor-bom                      = { group = "io.ktor",  name = "ktor-bom", version.ref = "ktor"  }
 
-ktor-clientAuth               = { group = "io.ktor",  name = "ktor-client-auth"                }
+ktorSerialization-kotlinxJson = { group = "io.ktor",  name = "ktor-serialization-kotlinx-json" }
-ktor-clientCIO                = { group = "io.ktor",  name = "ktor-client-cio"                 }
+
-ktor-clientContentNegotiation = { group = "io.ktor",  name = "ktor-client-content-negotiation" }
+ktorClient-core               = { group = "io.ktor",  name = "ktor-client-core"                }
-ktor-clientCore               = { group = "io.ktor",  name = "ktor-client-core"                }
+ktorClient-auth               = { group = "io.ktor",  name = "ktor-client-auth"                }
-ktor-clientEncoding           = { group = "io.ktor",  name = "ktor-client-encoding"            }
+ktorClient-cio                = { group = "io.ktor",  name = "ktor-client-cio"                 }
-ktor-clientLogging            = { group = "io.ktor",  name = "ktor-client-logging"             }
+ktorClient-contentNegotiation = { group = "io.ktor",  name = "ktor-client-content-negotiation" }
-ktor-clientResources          = { group = "io.ktor",  name = "ktor-client-resources"           }
+ktorClient-encoding           = { group = "io.ktor",  name = "ktor-client-encoding"            }
-ktor-clientWebsockets         = { group = "io.ktor",  name = "ktor-client-websockets"          }
+ktorClient-logging            = { group = "io.ktor",  name = "ktor-client-logging"             }
-ktor-serializationKotlinxJson = { group = "io.ktor",  name = "ktor-serialization-kotlinx-json" }
+ktorClient-resources          = { group = "io.ktor",  name = "ktor-client-resources"           }
+ktorClient-websockets         = { group = "io.ktor",  name = "ktor-client-websockets"          }
+
+ktorClient-testMock           = { group = "io.ktor",  name = "ktor-client-mock"                }
 
 
-ktor-serverCore               = { group = "io.ktor",  name = "ktor-server-core"                }
+ktorServer-core               = { group = "io.ktor",  name = "ktor-server-core"                }
-ktor-serverNetty              = { group = "io.ktor",  name = "ktor-server-netty"               }
+ktorServer-auth               = { group = "io.ktor",  name = "ktor-server-auth"                }
-ktor-serverContentNegotiaion  = { group = "io.ktor",  name = "ktor-server-content-negotiation" }
+ktorServer-authJwt            = { group = "io.ktor",  name = "ktor-server-auth-jwt"            }
-ktor-serverCors               = { group = "io.ktor",  name = "ktor-server-cors"                }
+ktorServer-autoHeadResponse   = { group = "io.ktor",  name = "ktor-server-auto-head-response"  }
-ktor-serverResources          = { group = "io.ktor",  name = "ktor-server-resources"           }
+ktorServer-cachingHeaders     = { group = "io.ktor",  name = "ktor-server-caching-headers"     }
-ktor-serverAutoHeadResponse   = { group = "io.ktor",  name = "ktor-server-auto-head-response"  }
+ktorServer-callId             = { group = "io.ktor",  name = "ktor-server-call-id"             }
-ktor-serverStatusPages        = { group = "io.ktor",  name = "ktor-server-status-pages"        }
+ktorServer-callLogging        = { group = "io.ktor",  name = "ktor-server-call-logging"        }
-ktor-serverAuth               = { group = "io.ktor",  name = "ktor-server-auth"                }
+ktorServer-compression        = { group = "io.ktor",  name = "ktor-server-compression"         }
-ktor-serverAuthJwt            = { group = "io.ktor",  name = "ktor-server-auth-jwt"            }
+ktorServer-conditionalHeaders = { group = "io.ktor",  name = "ktor-server-conditional-headers" }
-ktor-serverSessions           = { group = "io.ktor",  name = "ktor-server-sessions"            }
+ktorServer-contentNegotiaion  = { group = "io.ktor",  name = "ktor-server-content-negotiation" }
-ktor-serverDataConversion     = { group = "io.ktor",  name = "ktor-server-data-conversion"     }
+ktorServer-cors               = { group = "io.ktor",  name = "ktor-server-cors"                }
-ktor-serverDefaultHeaders     = { group = "io.ktor",  name = "ktor-server-default-headers"     }
+ktorServer-dataConversion     = { group = "io.ktor",  name = "ktor-server-data-conversion"     }
-ktor-serverCompression        = { group = "io.ktor",  name = "ktor-server-compression"         }
+ktorServer-defaultHeaders     = { group = "io.ktor",  name = "ktor-server-default-headers"     }
-ktor-serverCachingHeaders     = { group = "io.ktor",  name = "ktor-server-caching-headers"     }
+ktorServer-forwardedHeader    = { group = "io.ktor",  name = "ktor-server-forwarded-header"    }
-ktor-serverConditionalHeaders = { group = "io.ktor",  name = "ktor-server-conditional-headers" }
+ktorServer-hsts               = { group = "io.ktor",  name = "ktor-server-hsts"                }
-ktor-serverForwardedHeader    = { group = "io.ktor",  name = "ktor-server-forwarded-header"    }
+ktorServer-httpRedirect       = { group = "io.ktor",  name = "ktor-server-http-redirect"       }
-ktor-serverHsts               = { group = "io.ktor",  name = "ktor-server-hsts"                }
+#ktorServer-netty              = { group = "io.ktor",  name = "ktor-server-netty"               }
-ktor-serverHttpRedirect       = { group = "io.ktor",  name = "ktor-server-http-redirect"       }
+ktorServer-cio                = { group = "io.ktor",  name = "ktor-server-cio"                 }
-ktor-serverPartialContent     = { group = "io.ktor",  name = "ktor-server-partial-content"     }
+ktorServer-partialContent     = { group = "io.ktor",  name = "ktor-server-partial-content"     }
-ktor-serverWebsockets         = { group = "io.ktor",  name = "ktor-server-websockets"          }
+ktorServer-resources          = { group = "io.ktor",  name = "ktor-server-resources"           }
-ktor-serverCallLogging        = { group = "io.ktor",  name = "ktor-server-call-logging"        }
+ktorServer-sessions           = { group = "io.ktor",  name = "ktor-server-sessions"            }
-ktor-serverCallId             = { group = "io.ktor",  name = "ktor-server-call-id"             }
+ktorServer-statusPages        = { group = "io.ktor",  name = "ktor-server-status-pages"        }
+ktorServer-websockets         = { group = "io.ktor",  name = "ktor-server-websockets"          }
 
-ktor-network                  = { group = "io.ktor",  name = "ktor-network"                    }
+ktorServer-testHost           = { group = "io.ktor",  name = "ktor-server-test-host"           }
-ktor-networkTls               = { group = "io.ktor",  name = "ktor-network-tls"                }
+
+
+ktorNetwork-core              = { group = "io.ktor",  name = "ktor-network"                    }
+ktorNetwork-tls               = { group = "io.ktor",  name = "ktor-network-tls"                }
 
-ktor-test-clientMock          = { group = "io.ktor",  name = "ktor-client-mock"                }
-ktor-test-serverHost          = { group = "io.ktor",  name = "ktor-server-test-host"           }
 
 
 ### jackson ##

modules/events-processor/build.gradle.kts

@@ -0,0 +1,125 @@
+import dev.adamko.kafkatorio.task.KafkaConsumerGroupsTask
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+
+plugins {
+  dev.adamko.kafkatorio.lang.`kotlin-jvm`
+  kotlin("plugin.serialization")
+  application
+}
+
+
+description = """
+    Receive raw Factorio Events from Kafka and process them into targeted topics or data formats. 
+    
+    Factorio events -> Mod -> Kafka -> ***Factorio Events Processor*** -> processed events
+  """.trimIndent()
+
+val projectId: String by project.extra
+
+val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
+
+
+dependencies {
+  implementation(platform(projects.modules.versionsPlatform))
+
+  implementation(libs.kafka.streams)
+  testImplementation(libs.kafka.streamsTestUtils)
+
+  implementation(projects.modules.eventsSchema)
+
+//  implementation("com.ionspin.kotlin:bignum:0.3.3")
+
+  implementation(libs.kotlinx.serialization.core)
+  implementation(libs.kotlinx.serialization.json)
+//  implementation("org.jetbrains.kotlinx:kotlinx-serialization-protobuf")
+
+  implementation(libs.kotlinx.coroutines.core)
+
+  implementation(libs.bundles.http4k)
+  implementation(libs.http4k.format.kotlinx)
+
+
+  implementation(libs.bundles.logging)
+
+  implementation(libs.scrimage.core)
+
+  implementation(libs.okio.core)
+
+
+  //<editor-fold desc="Ktor">
+  implementation(libs.ktorSerialization.kotlinxJson)
+
+  with(libs.ktorServer) {
+    implementation(core)
+    implementation(callId)
+    implementation(callLogging)
+    implementation(contentNegotiaion)
+    implementation(cio)
+//    implementation(netty)
+    implementation(autoHeadResponse)
+    implementation(partialContent)
+    implementation(resources)
+    implementation(statusPages)
+    implementation(websockets)
+    implementation(compression)
+  }
+
+  with(libs.ktorNetwork) {
+    implementation(core)
+    implementation(tls)
+  }
+
+  //</editor-fold>
+
+
+  implementation(libs.simpleSyslogParser)
+
+  implementation(libs.kafka.kotkaStreams) {
+    isChanging = false
+  }
+
+  implementation(libs.bundles.hoplite)
+}
+
+
+application {
+  mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
+
+//  applicationDefaultJvmArgs += listOf(
+//    "-Dcom.sun.management.jmxremote.port=9186",
+//    "-Dcom.sun.management.jmxremote.rmi.port=9186",
+//    "-Dcom.sun.management.jmxremote.ssl=false",
+//    "-Dcom.sun.management.jmxremote.local.only=false",
+//    "-Dcom.sun.management.jmxremote.authenticate=false",
+//  )
+}
+
+tasks.withType<KotlinCompile>().configureEach {
+  kotlinOptions.freeCompilerArgs += listOf(
+    "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
+    "-opt-in=kotlinx.coroutines.FlowPreview",
+    "-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
+  )
+}
+
+
+val kafkaStateDirDelete by tasks.registering(Delete::class) {
+  group = project.name
+  mustRunAfter(kafkaConsumersDelete)
+  delete(kafkaStateDir)
+}
+
+val kafkaConsumersDelete by tasks.registering(KafkaConsumerGroupsTask.DeleteAll::class) {
+  group = project.name
+}
+
+//tasks.run.configure {
+//  dependsOn(kafkatorioEventsServerKafkaForceReset)
+//}
+
+idea {
+  module {
+    excludeDirs = excludeDirs + kafkaStateDir.asFile
+  }
+}

modules/events-schema/build.gradle.kts

@@ -11,6 +11,9 @@ plugins {
 }
 
 
+description = "shared data structures and utilities"
+
+
 kotlin {
 
   js(IR) {

modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/mapTile.kt

@@ -80,16 +80,33 @@ data class ServerMapChunkId(
 )
 
 
+/** `${serverDataDir}/servers/{serverId}/map/tiles/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png` */
 @Serializable
 @JvmInline
 value class TilePngFilename(
   val value: String,
 ) {
-  constructor(id: ServerMapChunkId) : this(buildString {
+  constructor(id: ServerMapChunkId) : this(
-    append("s${id.surfaceIndex}")
+    id.serverId,
-    append("/z${id.chunkSize.zoomLevel}")
+    id.surfaceIndex,
-    append("/x${id.chunkPosition.x}")
+    id.chunkSize.zoomLevel,
-    append("/y${id.chunkPosition.y}")
+    id.chunkPosition.x,
+    id.chunkPosition.y,
+  )
+
+  constructor(
+    serverId: FactorioServerId,
+    surfaceIndex: SurfaceIndex,
+    zoomLevel: Int,
+    chunkX: Int,
+    chunkY: Int,
+  ) : this(buildString {
+    append("servers/${serverId}")
+    append("/map/tiles")
+    append("/s${surfaceIndex}")
+    append("/z${zoomLevel}")
+    append("/x${chunkX}")
+    append("/y${chunkY}")
     append(".png")
   })
 }

modules/events-server/build.gradle.kts

@@ -9,11 +9,7 @@ plugins {
 }
 
 
-description = """
+description = "serves Kafkatorio content over the web"
-    Receive raw Factorio Events from Kafka and process them into targeted topics or data formats. 
-    
-    Factorio events -> Mod -> Kafka -> ***Factorio Events Processor*** -> processed events
-  """.trimIndent()
 
 val projectId: String by project.extra
 
@@ -39,19 +35,44 @@ dependencies {
   implementation(libs.bundles.http4k)
   implementation(libs.http4k.format.kotlinx)
 
+
   implementation(libs.bundles.logging)
 
   implementation(libs.scrimage.core)
 
   implementation(libs.okio.core)
 
-  implementation(libs.ktor.network)
+
-  implementation(libs.ktor.networkTls)
+  //<editor-fold desc="Ktor">
+  implementation(libs.ktorSerialization.kotlinxJson)
+
+  with(libs.ktorServer) {
+    implementation(core)
+    implementation(callId)
+    implementation(callLogging)
+    implementation(contentNegotiaion)
+    implementation(cio)
+//    implementation(netty)
+    implementation(autoHeadResponse)
+    implementation(partialContent)
+    implementation(resources)
+    implementation(statusPages)
+    implementation(websockets)
+    implementation(compression)
+  }
+
+  with(libs.ktorNetwork) {
+    implementation(core)
+    implementation(tls)
+  }
+
+  //</editor-fold>
+
 
   implementation(libs.simpleSyslogParser)
 
   implementation(libs.kafka.kotkaStreams) {
-    isChanging = true
+    isChanging = false
   }
 
   implementation(libs.bundles.hoplite)
@@ -59,7 +80,8 @@ dependencies {
 
 
 application {
-  mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
+//  mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
+  mainClass.set("dev.adamko.kafkatorio.server.KafkatorioEventsServerKt")
 
 //  applicationDefaultJvmArgs += listOf(
 //    "-Dcom.sun.management.jmxremote.port=9186",

modules/events-server/docker/Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /app
 ENV SERVER_PORT=8080
 EXPOSE $SERVER_PORT
 
-ENV TILE_DIR="/app/tiles"
+ENV SERVER_DATA_DIR="/app/data"
 ENV KAFKA_STREAMS_STATE_DIR="/app/kafka-state"
 VOLUME ${TILE_DIR} \
        ${KAFKA_STREAMS_STATE_DIR}

modules/events-server/docker/docker-compose.yml

@@ -14,7 +14,7 @@ services:
         DIST_DIR: "./build/distributions/"
     environment:
       KAFKA_BOOTSTRAP_SERVERS: "http://kafka:29092"
-      # KAFKA_STREAMS_STATE_DIR: "http://kafka:29092"
+      # KAFKA_STREAMS_STATE_DIR: ""
       SERVER_PORT: "8080"
     ports:
       - "12080:8080" # http and ws

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt

@@ -79,7 +79,7 @@ internal class KafkatorioTopology(
 
   private suspend fun saveTiles() {
     val builder = StreamsBuilder()
-    val topology = saveMapTiles(builder, appProps.tileDir)
+    val topology = saveMapTiles(builder, appProps.serverDataDir)
     launchTopology("saveTiles", topology)
   }
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt

@@ -9,7 +9,7 @@ import java.nio.file.Path
 // loaded by hoplite
 data class ApplicationProperties(
   val webPort: PortVal,
-  val tileDir: Path,
+  val serverDataDir: Path,
   val kafkaAdminConfig: Map<String, String>,
   val kafkaStreamsConfig: Map<String, String>,
 

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/PortVal.kt

@@ -17,6 +17,7 @@ value class PortVal(val value: Int) {
 
   companion object {
     val DEFAULT = PortVal(8080)
-    val VALID_RANGE: IntRange = 1..65535
+    val VALID_RANGE: IntRange
+      get() = (1..65535) // get() is required to prevent init ordering causing NPE
   }
 }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes/kxsJson.kt

@@ -2,20 +2,9 @@ package dev.adamko.kafkatorio.processor.serdes
 
 import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
 import kotlinx.serialization.json.Json
-import org.http4k.format.ConfigurableKotlinxSerialization
-import org.http4k.format.asConfigurable
-import org.http4k.format.withStandardMappings
 
 val jsonMapper: Json = Json {
   prettyPrint = true
   prettyPrintIndent = "  "
   serializersModule = jsonMapperKafkatorio.serializersModule
 }
-
-object KXS : ConfigurableKotlinxSerialization({
-  ignoreUnknownKeys = true
-  prettyPrint = true
-  prettyPrintIndent = "  "
-  asConfigurable().withStandardMappings().done()
-  serializersModule = jsonMapperKafkatorio.serializersModule
-})

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt

@@ -2,7 +2,6 @@ package dev.adamko.kafkatorio.processor.tileserver
 
 import dev.adamko.kafkatorio.processor.config.ApplicationProperties
 import java.security.MessageDigest
-import kotlin.io.path.absolutePathString
 import org.http4k.core.Filter
 import org.http4k.core.HttpHandler
 import org.http4k.core.MemoryBody
@@ -21,16 +20,19 @@ internal class WebMapTileServer(
   appProps: ApplicationProperties
 ) {
 
-  private val tilesLoader = ResourceLoader.Directory(appProps.tileDir.toFile().canonicalPath)
+  private val tilesLoader = ResourceLoader.Directory(appProps.serverDataDir.toFile().canonicalPath)
+
 
   private val filters = listOf(
     ETagChecker(),
   ).reduce(Filter::then)
 
+
   private val routes: RoutingHttpHandler = routes(
     "/tiles" bind GET to static(tilesLoader).withFilter(filters)
   )
 
+
   fun build(): HttpHandler {
     return routes
   }
@@ -69,6 +71,7 @@ internal class WebMapTileServer(
     }
   }
 
+
   companion object {
     private fun ByteArray.md5(): String? {
       return if (isEmpty()) {

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/saveMapTiles.kt

@@ -32,11 +32,11 @@ private const val WEB_MAP_TILE_SIZE_PX = 256
 
 
 /**
- * @param[tileDirectory] [dev.adamko.kafkatorio.processor.config.ApplicationProperties.webmapTileDir]
+ * @param[serverDataDir] [dev.adamko.kafkatorio.processor.config.ApplicationProperties.serverDataDir]
  */
 fun saveMapTiles(
   builder: StreamsBuilder,
-  tileDirectory: Path,
+  serverDataDir: Path,
 ): Topology {
 
   val subdividedMapChunkTilesDebounced: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
@@ -50,7 +50,7 @@ fun saveMapTiles(
     )
 
   val savedChunkFilenames: KStream<ServerMapChunkId, TilePngFilename> =
-    subdividedMapChunkTilesDebounced.saveChunkAsImage(tileDirectory)
+    subdividedMapChunkTilesDebounced.saveChunkAsImage(serverDataDir)
 
   savedChunkFilenames.broadcastSavedTileUpdates()
 
@@ -95,10 +95,10 @@ private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.saveChunkA
 @Synchronized
 private fun saveTile(
   filename: TilePngFilename,
-  tileDirectory: Path,
+  serverDataDir: Path,
   img: ImmutableImage,
 ) {
-  val chunkImageFile = tileDirectory.resolve(filename.value).toFile()
+  val chunkImageFile = serverDataDir.resolve(filename.value).toFile()
   if (chunkImageFile.parentFile.mkdirs()) {
     println("created new map tile parentFile directory ${chunkImageFile.absolutePath}")
   }

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/kafkatorioEventsServer.kt

@@ -0,0 +1,38 @@
+package dev.adamko.kafkatorio.server
+
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import io.ktor.http.HttpStatusCode
+import io.ktor.server.application.Application
+import io.ktor.server.application.install
+import io.ktor.server.cio.CIO
+import io.ktor.server.engine.embeddedServer
+import io.ktor.server.plugins.callloging.CallLogging
+import io.ktor.server.plugins.statuspages.StatusPages
+import io.ktor.server.response.respondText
+
+
+fun main() {
+  val appProps = ApplicationProperties.load()
+
+  embeddedServer(CIO, port = appProps.webPort.value) {
+
+    common()
+
+    webmapWebsocketServer()
+
+    webmapTileServer(appProps)
+
+    syslogSocketServer(appProps)
+
+  }.start(wait = true)
+}
+
+
+fun Application.common() {
+  install(StatusPages) {
+    exception<Throwable> { call, cause ->
+      call.respondText(text = "500: $cause", status = HttpStatusCode.InternalServerError)
+    }
+  }
+  install(CallLogging)
+}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/syslogSocketServer.kt

@@ -0,0 +1,152 @@
+package dev.adamko.kafkatorio.server
+
+import com.github.palindromicity.syslog.AllowableDeviations
+import com.github.palindromicity.syslog.SyslogParser
+import com.github.palindromicity.syslog.SyslogParserBuilder
+import com.github.palindromicity.syslog.SyslogSpecification
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.server.SyslogMsg.Companion.toSyslogMsg
+import io.ktor.network.selector.SelectorManager
+import io.ktor.network.sockets.aSocket
+import io.ktor.network.sockets.awaitClosed
+import io.ktor.network.sockets.openReadChannel
+import io.ktor.server.application.Application
+import io.ktor.utils.io.readUTF8Line
+import java.util.EnumSet
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.SharedFlow
+import kotlinx.coroutines.flow.asSharedFlow
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.yield
+
+
+fun Application.syslogSocketServer(appProps: ApplicationProperties) {
+  val server = SyslogSocketServer(appProps)
+
+  launch {
+    server.start()
+  }
+}
+
+
+class SyslogSocketServer(
+  appProps: ApplicationProperties,
+) {
+
+  private val selectorManager: SelectorManager = SelectorManager(Dispatchers.IO)
+
+
+  private val serverSocketTcp = aSocket(selectorManager)
+    .tcp()
+    .bind(
+      appProps.socketServerHost,
+      appProps.socketServerPort,
+    )
+
+
+  private val syslogParser: SyslogParser = SyslogParserBuilder()
+    .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION))
+    .forSpecification(SyslogSpecification.RFC_5424)
+    .build()
+
+
+  private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
+  val messages: SharedFlow<SyslogMsg>
+    get() = _messages.asSharedFlow()
+
+
+  suspend fun start(): Unit = coroutineScope {
+    println("[SyslogSocketServer] started listening: ${serverSocketTcp.localAddress}")
+
+    while (isActive) {
+      val socket = serverSocketTcp.accept()
+
+      launch {
+        val channel = socket.openReadChannel()
+        try {
+          while (socket.socketContext.isActive) {
+            val line = runCatching { channel.readUTF8Line() }.getOrNull()
+            if (line != null) {
+
+              println("[SyslogSocketServer] received: $line")
+
+              val syslog = syslogParser.parseLine(line).toSyslogMsg()
+
+              _messages.emit(syslog)
+            }
+
+            yield()
+          }
+        } catch (e: Exception) {
+          socket.awaitClosed()
+        }
+      }
+
+      yield()
+    }
+  }
+}
+
+
+data class SyslogMsg(
+  val src: Map<String, String?>
+) {
+
+  val message: String?
+    get() = src[SyslogFieldKeys.MESSAGE.field]
+
+  val headerAppName: String?
+    get() = src[SyslogFieldKeys.HEADER_APPNAME.field]
+
+  val headerHostName: String?
+    get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field]
+
+  val headerPri: String?
+    get() = src[SyslogFieldKeys.HEADER_PRI.field]
+
+  val headerPriSeverity: String?
+    get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field]
+
+  val headerPriFacility: String?
+    get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field]
+
+  val headerProcId: String?
+    get() = src[SyslogFieldKeys.HEADER_PROCID.field]
+
+  val headerTimestamp: String?
+    get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field]
+
+  val headerMsgId: String?
+    get() = src[SyslogFieldKeys.HEADER_MSGID.field]
+
+  val headerVersion: String?
+    get() = src[SyslogFieldKeys.HEADER_VERSION.field]
+
+  val structuredBase: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_BASE.field]
+
+  val structuredElementIdFmt: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field]
+
+  val structuredElementIdPnameFmt: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field]
+
+  val structuredElementIdPnamePattern: String?
+    get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field]
+
+  companion object {
+    fun Map<String, Any?>.toSyslogMsg(): SyslogMsg {
+      val map = entries.mapNotNull { (k, v) ->
+        when (v) {
+          is String -> k to v
+          else      -> null
+        }
+      }.toMap()
+      return SyslogMsg(map)
+    }
+  }
+}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/webmapTileServer.kt

@@ -0,0 +1,148 @@
+package dev.adamko.kafkatorio.server
+
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.schema.common.FactorioServerId
+import dev.adamko.kafkatorio.schema.common.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.schema.common.TilePngFilename
+import io.ktor.http.ContentType
+import io.ktor.http.HttpStatusCode
+import io.ktor.http.content.EntityTagVersion
+import io.ktor.http.content.versions
+import io.ktor.resources.Resource
+import io.ktor.serialization.kotlinx.json.json
+import io.ktor.server.application.Application
+import io.ktor.server.application.ApplicationCall
+import io.ktor.server.application.call
+import io.ktor.server.application.install
+import io.ktor.server.http.content.LastModifiedVersion
+import io.ktor.server.plugins.autohead.AutoHeadResponse
+import io.ktor.server.plugins.compression.Compression
+import io.ktor.server.plugins.compression.excludeContentType
+import io.ktor.server.plugins.compression.minimumSize
+import io.ktor.server.plugins.conditionalheaders.ConditionalHeaders
+import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
+import io.ktor.server.plugins.partialcontent.PartialContent
+import io.ktor.server.resources.Resources
+import io.ktor.server.resources.get
+import io.ktor.server.response.respond
+import io.ktor.server.response.respondFile
+import io.ktor.server.routing.routing
+import io.ktor.util.combineSafe
+import io.ktor.util.pipeline.PipelineContext
+import kotlin.io.path.Path
+import kotlinx.serialization.Serializable
+
+
+fun Application.webmapTileServer(appProps: ApplicationProperties) {
+
+  install(AutoHeadResponse)
+  install(Compression) {
+    default()
+    minimumSize(1024)
+    excludeContentType(ContentType.Video.Any)
+  }
+  install(ConditionalHeaders)
+  install(ContentNegotiation) {
+    json()
+  }
+  install(PartialContent)
+  install(Resources)
+
+  val server = WebmapTileServer(appProps)
+
+  routing {
+    with(server) {
+
+      get<FactorioServers.FactorioServer.MapData.Tile> { tile ->
+        handleTileRequest(tile)
+      }
+
+      get<FactorioServers> {
+
+      }
+    }
+  }
+}
+
+
+private class WebmapTileServer(
+  private val appProps: ApplicationProperties
+) {
+  private val serverDataDir by appProps::serverDataDir
+
+  suspend fun PipelineContext<Unit, ApplicationCall>.handleTileRequest(
+    tile: FactorioServers.FactorioServer.MapData.Tile
+  ) {
+
+    val tileFilename = TilePngFilename(
+      tile.serverId,
+      tile.surfaceIndex,
+      tile.zoomLevel,
+      tile.chunkX,
+      tile.chunkY,
+    )
+
+    val tileFile = serverDataDir.combineSafe(Path(tileFilename.value))
+
+    if (tileFile.exists()) {
+      call.respondFile(tileFile) {
+        versions = versions + listOf(
+          EntityTagVersion(tileFile.lastModified().hashCode().toString()),
+          LastModifiedVersion(tileFile.lastModified()),
+        )
+      }
+    } else {
+      call.respond(HttpStatusCode.NotFound)
+    }
+  }
+}
+
+
+@Serializable
+@Resource("servers")
+private class FactorioServers {
+
+  @Serializable
+  @Resource("{factorioServerId}")
+  data class FactorioServer(
+    val factorioServerId: FactorioServerId,
+  ) {
+
+    @Serializable
+    @Resource("map")
+    class MapData {
+
+      @Serializable
+      @Resource("tiles/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png")
+      data class Tile(
+        val parent: FactorioServer,
+        val surfaceIndex: SurfaceIndex,
+        val zoomLevel: Int,
+        val chunkX: Int,
+        val chunkY: Int,
+      ) {
+        val serverId: FactorioServerId by parent::factorioServerId
+      }
+
+      @Serializable
+      @Resource("buildings/s{surfaceIndex}")
+      data class Buildings(
+        val surfaceIndex: SurfaceIndex,
+        // a list of buildings... (Sealed interface, different types?)
+      )
+
+    }
+
+    @Serializable
+    @Resource("players")
+    class Players {
+
+      @Serializable
+      @Resource("{playerId}")
+      data class Id(
+        val playerIndex: PlayerIndex,
+      )
+    }
+  }
+}

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/webmapWebsocketServer.kt

@@ -0,0 +1,130 @@
+package dev.adamko.kafkatorio.server
+
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import io.ktor.server.application.Application
+import io.ktor.server.application.install
+import io.ktor.server.routing.routing
+import io.ktor.server.websocket.WebSockets
+import io.ktor.server.websocket.webSocket
+import io.ktor.websocket.DefaultWebSocketSession
+import io.ktor.websocket.Frame
+import io.ktor.websocket.WebSocketDeflateExtension
+import io.ktor.websocket.send
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.zip.Deflater
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.TimeSource
+import kotlinx.coroutines.channels.BufferOverflow
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
+import kotlinx.coroutines.flow.receiveAsFlow
+import kotlinx.coroutines.flow.update
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.job
+import kotlinx.coroutines.launch
+
+
+fun Application.webmapWebsocketServer() {
+  install(WebSockets) {
+    extensions {
+      install(WebSocketDeflateExtension) {
+        // Compression level to use for java.util.zip.Deflater
+        compressionLevel = Deflater.DEFAULT_COMPRESSION
+
+        // Prevent compressing small outgoing frames
+        compressIfBiggerThan(bytes = 4 * 1024)
+      }
+    }
+  }
+
+  val wsServer = WebmapWebsocketServer()
+
+  routing {
+
+    webSocket("/ws") {
+      wsServer.bindClient(this)
+    }
+  }
+}
+
+
+private class WebmapWebsocketServer {
+
+  private val startTime = TimeSource.Monotonic.markNow()
+
+  private val clients: MutableStateFlow<Set<WSConnection>> = MutableStateFlow(emptySet())
+
+  private val messagesToClients: MutableSharedFlow<String> = MutableSharedFlow(
+    extraBufferCapacity = Int.MAX_VALUE,
+    onBufferOverflow = BufferOverflow.DROP_OLDEST
+  )
+
+  private val messagesFromClients: MutableSharedFlow<Frame> = MutableSharedFlow(
+    extraBufferCapacity = Int.MAX_VALUE,
+    onBufferOverflow = BufferOverflow.DROP_OLDEST
+  )
+
+  suspend fun bindClient(session: DefaultWebSocketSession) {
+    val client = WSConnection(session)
+
+    clients.update { clients -> clients + client }
+
+    messagesToClients.onEach { msg ->
+      runCatching {
+        client.send(msg)
+      }
+    }.launchIn(client)
+
+    client.launch {
+      client.incoming.receiveAsFlow()
+        .collect(messagesFromClients)
+    }
+
+    client.launch {
+      while (isActive) {
+        runCatching {
+          client.send("elapsed ${startTime.elapsedNow()}")
+        }
+        delay(10.seconds)
+      }
+    }
+
+    client.coroutineContext.job.invokeOnCompletion {
+      log("client ${client.name} completed")
+      clients.update { clients -> clients - client }
+    }
+  }
+
+
+  suspend fun sendMessageToClients(msg: String) {
+    log("sending message to clients: $msg")
+    messagesToClients.emit(msg)
+  }
+}
+
+
+private data class WSConnection(
+  private val session: DefaultWebSocketSession,
+) : DefaultWebSocketSession by session {
+  val name: String = "user${lastId.getAndIncrement()}"
+
+  override suspend fun send(frame: Frame): Unit = runCatching {
+    super.send(frame)
+  }.fold(
+    onSuccess = { log("sent message to client $name: $frame") },
+    onFailure = { log("failed to send message to client $name: $frame") }
+  )
+
+  override fun equals(other: Any?): Boolean = name == other
+  override fun hashCode(): Int = name.hashCode()
+
+  companion object {
+    private val lastId = AtomicInteger(0)
+  }
+}
+
+
+private fun log(msg: String) = println("[WebmapWebsocketServer] $msg")

modules/events-server/src/main/resources/config.yml

@@ -1,7 +1,7 @@
 # Factorio Events Processor config
 
 web-port: ${SERVER_PORT:-8080}
-tile-dir: ${TILE_DIR:-./tile-dir}
+servers-data-dir: ${SERVER_DATA_DIR:-./server-data}
 
 
 socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}

settings.gradle.kts

@@ -3,6 +3,7 @@ rootProject.name = "kafkatorio"
 include(
   ":modules:events-mod",
   ":modules:events-schema",
+  ":modules:events-processor",
   ":modules:events-server",
 
   ":modules:infra-factorio-client",