big ol' refactor to split up the events-server and events-processors

modules/events-server-web/build.gradle.kts

@@ -0,0 +1,91 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+
+plugins {
+  dev.adamko.kafkatorio.lang.`kotlin-jvm`
+  kotlin("plugin.serialization")
+  application
+}
+
+
+description = "serves Kafkatorio content over web APIs"
+
+val projectId: String by project.extra
+
+val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka")
+
+
+dependencies {
+  implementation(platform(projects.modules.versionsPlatform))
+
+  implementation(libs.kafka.streams)
+  testImplementation(libs.kafka.streamsTestUtils)
+
+  implementation(projects.modules.eventsLibrary)
+  implementation(projects.modules.eventsProcessorCore)
+
+  implementation(libs.kotlinxSerialization.core)
+  implementation(libs.kotlinxSerialization.json)
+
+  implementation(libs.kotlinx.coroutines.core)
+
+  implementation(libs.bundles.logging)
+
+  implementation(libs.scrimage.core)
+
+  implementation(libs.okio.core)
+
+  //<editor-fold desc="Ktor">
+  implementation(libs.ktorSerialization.kotlinxJson)
+
+  with(libs.ktorServer) {
+    implementation(auth)
+    implementation(authJwt)
+    implementation(core)
+    implementation(callId)
+    implementation(callLogging)
+    implementation(contentNegotiaion)
+    implementation(cio)
+    implementation(autoHeadResponse)
+    implementation(partialContent)
+    implementation(resources)
+    implementation(statusPages)
+    implementation(websockets)
+    implementation(compression)
+  }
+
+  with(libs.ktorNetwork) {
+    implementation(core)
+    implementation(tls)
+  }
+  //</editor-fold>
+
+  implementation(libs.simpleSyslogParser)
+
+//  implementation(libs.nimbusJoseJwt)
+
+  implementation(libs.kafka.kotkaStreams)
+
+  implementation(libs.bundles.hoplite)
+}
+
+
+application {
+  mainClass.set("dev.adamko.kafkatorio.server.web.MainKt")
+}
+
+
+tasks.withType<KotlinCompile>().configureEach {
+  kotlinOptions.freeCompilerArgs += listOf(
+    "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
+    "-opt-in=kotlinx.coroutines.FlowPreview",
+    "-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
+  )
+}
+
+
+val runEventsWebServer by tasks.registering {
+  group = rootProject.name
+
+  dependsOn(tasks.run)
+}

modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/config/kxsJson.kt

@@ -0,0 +1,11 @@
+package dev.adamko.kafkatorio.server.web.config
+
+import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
+import kotlinx.serialization.json.Json
+
+
+internal val jsonMapper: Json = Json {
+  prettyPrint = true
+  prettyPrintIndent = "  "
+  serializersModule = jsonMapperKafkatorio.serializersModule
+}

modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/main.kt

@@ -0,0 +1,61 @@
+package dev.adamko.kafkatorio.server.web
+
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.processor.core.launchTopology
+import dev.adamko.kafkatorio.server.processor.topology.broadcastToWebsocket
+import dev.adamko.kafkatorio.server.processor.topology.saveMapTiles
+import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.seconds
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.launch
+import org.apache.kafka.clients.producer.ProducerConfig
+import org.apache.kafka.streams.StreamsBuilder
+import org.apache.kafka.streams.StreamsConfig
+
+suspend fun main(): Unit = coroutineScope {
+  val appProps = ApplicationProperties.load()
+  val wsServer = WebmapWebsocketServer()
+
+  launch {
+    websocketBroadcaster(wsServer)
+  }
+
+//  launch {
+//    broadcastToWebsocket(wsServer, StreamsBuilder())
+//  }
+
+  launch {
+    saveTiles(appProps)
+  }
+
+  launch {
+    startWebServer(
+      appProps = appProps,
+      wsServer = wsServer,
+    )
+  }
+}
+
+private suspend fun websocketBroadcaster(wsServer: WebmapWebsocketServer) {
+  val builder = StreamsBuilder()
+
+  broadcastToWebsocket(wsServer, builder)
+
+  val topology = builder.build()
+
+  launchTopology(
+    "websocketBroadcaster", topology, mapOf(
+      StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
+      StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
+    )
+  )
+}
+
+private suspend fun saveTiles(
+  appProps: ApplicationProperties
+) {
+  val builder = StreamsBuilder()
+  saveMapTiles(builder, appProps.serverDataDir)
+  launchTopology("saveTiles", builder.build())
+}

modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/factorioServersRoutes.kt

@@ -0,0 +1,129 @@
+package dev.adamko.kafkatorio.server.web.rest
+
+import dev.adamko.kafkatorio.schema.common.FactorioServerId
+import dev.adamko.kafkatorio.schema.common.PlayerIndex
+import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
+import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
+import dev.adamko.kafkatorio.schema.common.SurfaceIndex
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.server.web.config.jsonMapper
+import io.ktor.http.HttpStatusCode
+import io.ktor.http.content.EntityTagVersion
+import io.ktor.http.content.versions
+import io.ktor.resources.Resource
+import io.ktor.server.application.ApplicationCall
+import io.ktor.server.application.call
+import io.ktor.server.http.content.LastModifiedVersion
+import io.ktor.server.resources.get
+import io.ktor.server.response.respond
+import io.ktor.server.response.respondFile
+import io.ktor.server.routing.Routing
+import io.ktor.util.combineSafe
+import io.ktor.util.pipeline.PipelineContext
+import kotlin.io.path.Path
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.encodeToJsonElement
+
+
+internal fun Routing.factorioServersRoutes(
+  appProps: ApplicationProperties,
+) {
+
+  get<FactorioServers> {
+    val serverIds = appProps.kafkatorioServers.values.toSet().sortedBy { it.id }
+    val serverIdsJson = jsonMapper.encodeToJsonElement(serverIds)
+    call.respond(serverIdsJson)
+  }
+
+  get<FactorioServer.MapData.Tile> { tile ->
+    handleTileRequest(tile, appProps)
+  }
+}
+
+
+private suspend fun PipelineContext<Unit, ApplicationCall>.handleTileRequest(
+  tile: FactorioServer.MapData.Tile,
+  appProps: ApplicationProperties,
+) {
+
+  val tileFilename = ServerMapTilePngFilename(
+    tile.serverId,
+    tile.layer,
+    tile.surfaceIndex,
+    tile.zoomLevel,
+    tile.chunkX,
+    tile.chunkY,
+  )
+
+  val tileFile = appProps.serverDataDir.combineSafe(Path(tileFilename.value))
+
+  if (tileFile.exists()) {
+    call.respondFile(tileFile) {
+      versions = versions + listOf(
+        EntityTagVersion(tileFile.lastModified().hashCode().toString()),
+        LastModifiedVersion(tileFile.lastModified()),
+      )
+    }
+  } else {
+    // It's not a client or server error to request a non-existent tile, it just doesn't exist
+    // (yet). So return 304.
+    call.respond(HttpStatusCode.NotModified)
+  }
+}
+
+
+@Serializable
+@Resource("kafkatorio/data/servers")
+private class FactorioServers
+
+
+@Serializable
+@Resource("kafkatorio/data/servers/{factorioServerId}")
+data class FactorioServer(
+  val factorioServerId: FactorioServerId,
+) {
+
+  @Serializable
+  @Resource("map")
+  data class MapData(
+    val server: FactorioServer,
+  ) {
+
+    @Serializable
+    @Resource("layers/{layer}/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png")
+    data class Tile(
+      val mapData: MapData,
+
+      val layer: ServerMapTileLayer,
+      val surfaceIndex: SurfaceIndex,
+      val zoomLevel: Int,
+      val chunkX: Int,
+      val chunkY: Int,
+    )
+
+    @Serializable
+    @Resource("buildings/s{surfaceIndex}")
+    data class Buildings(
+      val mapData: MapData,
+      val surfaceIndex: SurfaceIndex,
+      // a list of buildings... (Sealed interface, different types?)
+    )
+  }
+
+  @Serializable
+  @Resource("players")
+  data class Players(
+    val server: FactorioServer,
+  )
+
+  @Serializable
+  @Resource("{playerId}")
+  data class Player(
+    val server: FactorioServer,
+    val playerIndex: PlayerIndex,
+  )
+}
+
+
+private val FactorioServer.MapData.Tile.serverId: FactorioServerId
+  get() = mapData.server.factorioServerId

modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/restApi.kt

@@ -0,0 +1,56 @@
+package dev.adamko.kafkatorio.server.web.rest
+
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import io.ktor.http.ContentType
+import io.ktor.http.HttpStatusCode
+import io.ktor.http.isSuccess
+import io.ktor.serialization.kotlinx.json.json
+import io.ktor.server.application.Application
+import io.ktor.server.application.install
+import io.ktor.server.application.log
+import io.ktor.server.plugins.autohead.AutoHeadResponse
+import io.ktor.server.plugins.callloging.CallLogging
+import io.ktor.server.plugins.compression.Compression
+import io.ktor.server.plugins.compression.excludeContentType
+import io.ktor.server.plugins.compression.minimumSize
+import io.ktor.server.plugins.conditionalheaders.ConditionalHeaders
+import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
+import io.ktor.server.plugins.partialcontent.PartialContent
+import io.ktor.server.plugins.statuspages.StatusPages
+import io.ktor.server.resources.Resources
+import io.ktor.server.response.respondText
+import io.ktor.server.routing.routing
+
+
+internal fun Application.webmapTileServer(appProps: ApplicationProperties) {
+
+  install(AutoHeadResponse)
+  install(CallLogging) {
+    filter { call ->
+      // only log failures
+      !(call.response.status()?.isSuccess() ?: false)
+    }
+  }
+  install(Compression) {
+    default()
+    minimumSize(1024)
+    excludeContentType(ContentType.Video.Any)
+  }
+  install(ConditionalHeaders)
+  install(ContentNegotiation) {
+    json()
+  }
+  install(PartialContent)
+  install(Resources)
+
+  install(StatusPages) {
+    exception<Throwable> { call, cause ->
+      this@webmapTileServer.log.error("$call caused exception - returning 500", cause)
+      call.respondText(text = "500: $cause", status = HttpStatusCode.InternalServerError)
+    }
+  }
+
+  routing {
+    factorioServersRoutes(appProps)
+  }
+}

modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/webServer.kt

@@ -0,0 +1,28 @@
+package dev.adamko.kafkatorio.server.web
+
+
+import dev.adamko.kafkatorio.processor.config.ApplicationProperties
+import dev.adamko.kafkatorio.server.web.rest.webmapTileServer
+import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
+import dev.adamko.kafkatorio.server.web.websocket.webmapWebsocketServer
+import io.ktor.server.cio.CIO
+import io.ktor.server.engine.embeddedServer
+import kotlinx.coroutines.coroutineScope
+
+
+internal suspend fun startWebServer(
+  appProps: ApplicationProperties,
+  wsServer: WebmapWebsocketServer,
+): Unit = coroutineScope {
+
+  embeddedServer(
+    factory = CIO,
+    port = appProps.webPort.value,
+  ) {
+
+    webmapWebsocketServer(wsServer)
+
+    webmapTileServer(appProps)
+
+  }.start(wait = true)
+}

modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/WebmapWebsocketServer.kt

@@ -0,0 +1,130 @@
+package dev.adamko.kafkatorio.server.web.websocket
+
+import dev.adamko.kafkatorio.server.processor.topology.WebsocketBroadcaster
+import io.ktor.websocket.CloseReason
+import io.ktor.websocket.DefaultWebSocketSession
+import io.ktor.websocket.Frame
+import io.ktor.websocket.send
+import java.util.concurrent.atomic.AtomicInteger
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.TimeSource
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.channels.BufferOverflow
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
+import kotlinx.coroutines.flow.receiveAsFlow
+import kotlinx.coroutines.flow.update
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.job
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.yield
+
+
+class WebmapWebsocketServer : WebsocketBroadcaster {
+
+  private val startTime = TimeSource.Monotonic.markNow()
+
+  private val clients: MutableStateFlow<List<WSConnection>> = MutableStateFlow(emptyList())
+
+  private val messagesToClients: MutableSharedFlow<String> = MutableSharedFlow(
+    extraBufferCapacity = Int.MAX_VALUE,
+    onBufferOverflow = BufferOverflow.DROP_OLDEST
+  )
+
+  private val messagesFromClients: MutableSharedFlow<Frame> = MutableSharedFlow(
+    extraBufferCapacity = Int.MAX_VALUE,
+    onBufferOverflow = BufferOverflow.DROP_OLDEST
+  )
+
+
+  suspend fun bindClient(session: DefaultWebSocketSession): Unit = coroutineScope {
+    val client = WSConnection(session)
+
+    clients.update { clients -> clients + client }
+
+    client.coroutineContext.job.invokeOnCompletion { e ->
+      log("client ${client.name} completed ${e ?: "successfully"}")
+      clients.update { clients ->
+        (clients - client).filter { it.isActive }
+      }
+    }
+
+    messagesToClients.onEach { msg ->
+      runCatching {
+        if (client.isActive) {
+          client.outgoing.send(Frame.Text(msg))
+        }
+      }.onFailure { e ->
+        if (e is CancellationException) throw e
+      }
+    }.launchIn(client)
+
+    client.launch {
+      client.incoming.receiveAsFlow().collect(messagesFromClients)
+    }
+
+    client.launch {
+      while (isActive) {
+        runCatching {
+          client.send("elapsed ${startTime.elapsedNow()}")
+        }.onFailure { e ->
+          if (e is CancellationException) throw e
+        }
+        delay(10.seconds)
+        yield()
+      }
+    }
+
+    log("client ${client.name} is up and running...")
+    val closedReason: CloseReason? = client.closeReason.await()
+
+    log("client ${client.name} closed ${closedReason ?: "successfully"}")
+
+    coroutineContext.cancel()
+  }
+
+
+  override suspend fun sendMessageToClients(message: String) {
+    log("sending message to clients: $message")
+    messagesToClients.emit(message)
+  }
+
+
+  private data class WSConnection(
+    private val session: DefaultWebSocketSession,
+  ) : DefaultWebSocketSession by session {
+    val name: String = "user${lastId.getAndIncrement()}"
+
+    override suspend fun send(frame: Frame): Unit = runCatching {
+      super.send(frame)
+    }.fold(
+      onSuccess = {
+//        log("sent message to client $name: $frame")
+      },
+      onFailure = { e ->
+        if (e is CancellationException) throw e
+
+        log("failed to send message to client $name: $e")
+      }
+    )
+
+    override fun equals(other: Any?): Boolean = name == other
+    override fun hashCode(): Int = name.hashCode()
+
+    companion object {
+      private val lastId = AtomicInteger(0)
+    }
+  }
+
+
+  companion object {
+
+    private fun log(msg: String) = println("[WebmapWebsocketServer] $msg")
+
+  }
+}

modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/wsServer.kt

@@ -0,0 +1,34 @@
+package dev.adamko.kafkatorio.server.web.websocket
+
+import io.ktor.server.application.Application
+import io.ktor.server.application.install
+import io.ktor.server.routing.routing
+import io.ktor.server.websocket.WebSockets
+import io.ktor.server.websocket.webSocket
+import io.ktor.websocket.WebSocketDeflateExtension
+import java.util.zip.Deflater
+
+
+fun Application.webmapWebsocketServer(
+  wsServer: WebmapWebsocketServer,
+) {
+  install(WebSockets) {
+    extensions {
+      install(WebSocketDeflateExtension) {
+        // Compression level to use for java.util.zip.Deflater
+        compressionLevel = Deflater.DEFAULT_COMPRESSION
+
+        // Prevent compressing small outgoing frames
+        compressIfBiggerThan(bytes = 4 * 1024)
+      }
+    }
+  }
+
+  routing {
+    // TODO only send a ws client message for a single Factorio-sever. Make the URL server specific.
+    //      /kafkatorio/data/servers/{serverId}/ws
+    webSocket("kafkatorio/ws") {
+      wsServer.bindClient(this)
+    }
+  }
+}