lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

finish migrating to ktor!

Browse source 
- remove http4k
- renamed 'schema' subproject to 'library', because it contains more than just data models
- tiles are now hosted per factorio-server
- changed api base to `kafkatorio/data`
- tidied up some Gradle conf
- remove 'events-processor' subproject - I don't want to set up loads of microservices right now
This commit is contained in:
Adam 2022-07-02 08:45:21 +02:00
parent 8a90c4fb5e
commit c0c82e7c9c
80 changed files with 1878 additions and 1280 deletions
Download patch file Download diff file Expand all files Collapse all files

814
gradle/kotlin-js-store/yarn.lock
View file

File diff suppressed because it is too large Load diff

2
gradle/libs.versions.toml
View file

@ -153,7 +153,7 @@ ktorServer-defaultHeaders = { group = "io.ktor", name = "ktor-server-defaul
ktorServer-forwardedHeader = { group = "io.ktor", name = "ktor-server-forwarded-header" } ktorServer-forwardedHeader = { group = "io.ktor", name = "ktor-server-forwarded-header" }
ktorServer-hsts = { group = "io.ktor", name = "ktor-server-hsts" } ktorServer-hsts = { group = "io.ktor", name = "ktor-server-hsts" }
ktorServer-httpRedirect = { group = "io.ktor", name = "ktor-server-http-redirect" } ktorServer-httpRedirect = { group = "io.ktor", name = "ktor-server-http-redirect" }
#ktorServer-netty = { group = "io.ktor", name = "ktor-server-netty" } ktorServer-netty = { group = "io.ktor", name = "ktor-server-netty" }
ktorServer-cio = { group = "io.ktor", name = "ktor-server-cio" } ktorServer-cio = { group = "io.ktor", name = "ktor-server-cio" }
ktorServer-partialContent = { group = "io.ktor", name = "ktor-server-partial-content" } ktorServer-partialContent = { group = "io.ktor", name = "ktor-server-partial-content" }
ktorServer-resources = { group = "io.ktor", name = "ktor-server-resources" } ktorServer-resources = { group = "io.ktor", name = "ktor-server-resources" }

6
modules/events-schema/build.gradle.kts → modules/events-library/build.gradle.kts
View file

@ -51,6 +51,12 @@ kotlin {
implementation(libs.kotlinx.serialization.core) implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json) implementation(libs.kotlinx.serialization.json)
api("dev.adamko.kxstsgen:kxs-ts-gen-core:0.1.3") api("dev.adamko.kxstsgen:kxs-ts-gen-core:0.1.3")
implementation(dependencies.platform(libs.okio.bom))
implementation(libs.okio.core)
implementation(dependencies.platform(libs.kotlinx.coroutines.bom))
implementation(libs.kotlinx.coroutines.core)
} }
} }

3
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/json.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/library/json.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.schema package dev.adamko.kafkatorio.library
import kotlin.reflect.KClass import kotlin.reflect.KClass
import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.DeserializationStrategy
@ -24,7 +24,6 @@ val jsonMapperKafkatorio = Json {
serializersModule = SerializersModule { serializersModule = SerializersModule {
contextual(List::class) { args -> FactorioJsonListSerializer(args[0]) } contextual(List::class) { args -> FactorioJsonListSerializer(args[0]) }
} }
} }

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes/kxsBinary.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/library/kxsBinary.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.processor.serdes package dev.adamko.kafkatorio.library
import kotlinx.serialization.BinaryFormat import kotlinx.serialization.BinaryFormat
import kotlinx.serialization.DeserializationStrategy import kotlinx.serialization.DeserializationStrategy

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/colours.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/colours.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioIds.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioIds.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioPrototypes.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioPrototypes.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/mapTile.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/mapTile.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/positions.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/positions.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/eventServerPacket.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/eventServerPacket.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketInstant.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketInstant.kt
View file

0
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketKeyed.kt → modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketKeyed.kt
View file

3
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/ColourHexTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/ColourHexTest.kt
View file

@ -1,10 +1,10 @@
package dev.adamko.kafkatorio.events.schema package dev.adamko.kafkatorio.events.schema
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
import dev.adamko.kafkatorio.schema.common.Colour import dev.adamko.kafkatorio.schema.common.Colour
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.toHex import dev.adamko.kafkatorio.schema.common.toHex
import dev.adamko.kafkatorio.schema.common.toPercentile import dev.adamko.kafkatorio.schema.common.toPercentile
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import io.kotest.assertions.withClue import io.kotest.assertions.withClue
import io.kotest.core.spec.style.BehaviorSpec import io.kotest.core.spec.style.BehaviorSpec
import io.kotest.matchers.floats.shouldBeWithinPercentageOf import io.kotest.matchers.floats.shouldBeWithinPercentageOf
@ -13,6 +13,7 @@ import io.kotest.matchers.shouldBe
import io.kotest.property.checkAll import io.kotest.property.checkAll
import kotlinx.serialization.decodeFromString import kotlinx.serialization.decodeFromString
class ColourHexTest : BehaviorSpec({ class ColourHexTest : BehaviorSpec({
Given("colour json") { Given("colour json") {

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/MapChunkUpdateTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/MapChunkUpdateTest.kt
View file

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/PrototypesTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/PrototypesTest.kt
View file

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/UpdatePacketsTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/UpdatePacketsTest.kt
View file

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapChunkPositionBoundsTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapChunkPositionBoundsTest.kt
View file

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapChunkPositionTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapChunkPositionTest.kt
View file

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTileDictionaryTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTileDictionaryTest.kt
View file

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTilePositionTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTilePositionTest.kt
View file

0
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/packets/EntityUpdateTest.kt → modules/events-library/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/packets/EntityUpdateTest.kt
View file

0
modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/ValueClassTransformer.kt → modules/events-library/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/ValueClassTransformer.kt
View file

0
modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts.kt → modules/events-library/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts.kt
View file

0
modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts2.kt → modules/events-library/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts2.kt
View file

0
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/FileEventsFlow.kt → modules/events-library/src/jvmMain/kotlin/dev/adamko/kafkatorio/library/FileEventsFlow.kt
View file

19
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/KxsBinaryTests.kt → modules/events-library/src/jvmTest/kotlin/dev/adamko/kafkatorio/library/KxsBinaryTests.kt
View file

@ -1,15 +1,12 @@
package dev.adamko.kafkatorio.processor.misc package dev.adamko.kafkatorio.library
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
import dev.adamko.kafkatorio.processor.topology.ServerMapChunkTiles
import dev.adamko.kafkatorio.schema.common.ChunkSize import dev.adamko.kafkatorio.schema.common.ChunkSize
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.MapChunkPosition import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapTilePosition import dev.adamko.kafkatorio.schema.common.MapTilePosition
import dev.adamko.kafkatorio.schema.common.SurfaceIndex import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import io.kotest.assertions.withClue import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.collections.shouldContainExactly import io.kotest.matchers.collections.shouldContainExactly
@ -42,6 +39,7 @@ import io.kotest.property.arbitrary.uInt
import io.kotest.property.arbitrary.uLong import io.kotest.property.arbitrary.uLong
import io.kotest.property.arbitrary.uShort import io.kotest.property.arbitrary.uShort
import io.kotest.property.checkAll import io.kotest.property.checkAll
import kotlinx.serialization.Serializable
import kotlinx.serialization.builtins.serializer import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.decodeFromByteArray import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.decodeFromString import kotlinx.serialization.decodeFromString
@ -316,4 +314,15 @@ class KxsBinaryTests : FunSpec({
} }
} }
} }
}) }) {
@Serializable
data class ServerMapChunkTiles<Data>(
val chunkId: ServerMapChunkId,
val map: Map<MapTilePosition, Data>,
) {
operator fun plus(other: ServerMapChunkTiles<Data>) = copy(map = map + other.map)
}
}

5
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/serdes/MapTileDictionaryTest.kt → modules/events-library/src/jvmTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTileDictionaryTest.kt
View file

@ -1,7 +1,6 @@
package dev.adamko.kafkatorio.processor.serdes package dev.adamko.kafkatorio.schema.common
import dev.adamko.kafkatorio.schema.common.MapTileDictionary import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.schema.common.PrototypeName
import io.kotest.core.spec.style.FunSpec import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.equality.shouldBeEqualToComparingFields import io.kotest.matchers.equality.shouldBeEqualToComparingFields
import kotlinx.serialization.decodeFromByteArray import kotlinx.serialization.decodeFromByteArray

2
modules/events-mod/build.gradle.kts
View file

@ -63,7 +63,7 @@ val typescriptEventsSchema: Configuration by configurations.creating {
dependencies { dependencies {
typescriptEventsSchema(projects.modules.eventsSchema) typescriptEventsSchema(projects.modules.eventsLibrary)
} }

125
modules/events-processor/build.gradle.kts
View file

@ -1,125 +0,0 @@
import dev.adamko.kafkatorio.task.KafkaConsumerGroupsTask
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
description = """
Receive raw Factorio Events from Kafka and process them into targeted topics or data formats.
Factorio events -> Mod -> Kafka -> ***Factorio Events Processor*** -> processed events
""".trimIndent()
val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
dependencies {
implementation(platform(projects.modules.versionsPlatform))
implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsSchema)
// implementation("com.ionspin.kotlin:bignum:0.3.3")
implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json)
// implementation("org.jetbrains.kotlinx:kotlinx-serialization-protobuf")
implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.http4k)
implementation(libs.http4k.format.kotlinx)
implementation(libs.bundles.logging)
implementation(libs.scrimage.core)
implementation(libs.okio.core)
//<editor-fold desc="Ktor">
implementation(libs.ktorSerialization.kotlinxJson)
with(libs.ktorServer) {
implementation(core)
implementation(callId)
implementation(callLogging)
implementation(contentNegotiaion)
implementation(cio)
// implementation(netty)
implementation(autoHeadResponse)
implementation(partialContent)
implementation(resources)
implementation(statusPages)
implementation(websockets)
implementation(compression)
}
with(libs.ktorNetwork) {
implementation(core)
implementation(tls)
}
//</editor-fold>
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams) {
isChanging = false
}
implementation(libs.bundles.hoplite)
}
application {
mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
// applicationDefaultJvmArgs += listOf(
// "-Dcom.sun.management.jmxremote.port=9186",
// "-Dcom.sun.management.jmxremote.rmi.port=9186",
// "-Dcom.sun.management.jmxremote.ssl=false",
// "-Dcom.sun.management.jmxremote.local.only=false",
// "-Dcom.sun.management.jmxremote.authenticate=false",
// )
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
"-opt-in=kotlinx.coroutines.FlowPreview",
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}
val kafkaStateDirDelete by tasks.registering(Delete::class) {
group = project.name
mustRunAfter(kafkaConsumersDelete)
delete(kafkaStateDir)
}
val kafkaConsumersDelete by tasks.registering(KafkaConsumerGroupsTask.DeleteAll::class) {
group = project.name
}
//tasks.run.configure {
// dependsOn(kafkatorioEventsServerKafkaForceReset)
//}
idea {
module {
excludeDirs = excludeDirs + kafkaStateDir.asFile
}
}

2
modules/events-server/.gitignore vendored
View file

@ -1,2 +1,2 @@
kafka-state/ kafka-state/
kafkatorio-tiles/ server-data/

23
modules/events-server/build.gradle.kts
View file

@ -1,5 +1,6 @@
import dev.adamko.kafkatorio.task.KafkaConsumerGroupsTask import dev.adamko.kafkatorio.task.KafkaConsumerGroupsTask
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.utils.addToStdlib.applyIf
plugins { plugins {
@ -22,9 +23,7 @@ dependencies {
implementation(libs.kafka.streams) implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils) testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsSchema) implementation(projects.modules.eventsLibrary)
// implementation("com.ionspin.kotlin:bignum:0.3.3")
implementation(libs.kotlinx.serialization.core) implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json) implementation(libs.kotlinx.serialization.json)
@ -32,17 +31,12 @@ dependencies {
implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.http4k)
implementation(libs.http4k.format.kotlinx)
implementation(libs.bundles.logging) implementation(libs.bundles.logging)
implementation(libs.scrimage.core) implementation(libs.scrimage.core)
implementation(libs.okio.core) implementation(libs.okio.core)
//<editor-fold desc="Ktor"> //<editor-fold desc="Ktor">
implementation(libs.ktorSerialization.kotlinxJson) implementation(libs.ktorSerialization.kotlinxJson)
@ -81,15 +75,7 @@ dependencies {
application { application {
// mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt") // mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
mainClass.set("dev.adamko.kafkatorio.server.KafkatorioEventsServerKt") mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
// applicationDefaultJvmArgs += listOf(
// "-Dcom.sun.management.jmxremote.port=9186",
// "-Dcom.sun.management.jmxremote.rmi.port=9186",
// "-Dcom.sun.management.jmxremote.ssl=false",
// "-Dcom.sun.management.jmxremote.local.only=false",
// "-Dcom.sun.management.jmxremote.authenticate=false",
// )
} }
tasks.withType<KotlinCompile>().configureEach { tasks.withType<KotlinCompile>().configureEach {
@ -111,9 +97,6 @@ val kafkaConsumersDelete by tasks.registering(KafkaConsumerGroupsTask.DeleteAll:
group = project.name group = project.name
} }
//tasks.run.configure {
// dependsOn(kafkatorioEventsServerKafkaForceReset)
//}
idea { idea {
module { module {

58
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt
View file

@ -1,58 +0,0 @@
package dev.adamko.kafkatorio.processor
import dev.adamko.kafkatorio.processor.admin.KafkatorioKafkaAdmin
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer
import dev.adamko.kafkatorio.processor.tileserver.WebMapTileServer
import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer
import dev.adamko.kafkatorio.processor.tileserver.webServer
import kotlinx.coroutines.awaitCancellation
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
import org.http4k.server.Http4kServer
suspend fun main() {
val appProps = ApplicationProperties.load()
val admin = KafkatorioKafkaAdmin(appProps)
admin.createKafkatorioTopics()
val syslogServer = SyslogSocketServer(appProps)
val wsServer = WebsocketServer()
val tileServer = WebMapTileServer(appProps)
val topology = KafkatorioTopology(appProps, wsServer, syslogServer)
coroutineScope {
launch {
syslogServer.start()
}
launch {
topology.start()
println("launched KafkatorioTopology")
awaitCancellation()
}
launch {
val webServer: Http4kServer = webServer(tileServer, wsServer, appProps)
currentCoroutineContext().job.invokeOnCompletion {
println("webserver stopping. Cause: $it")
webServer.stop()
}
webServer.start()
println("launched webserver port:${webServer.port()}")
awaitCancellation()
}
}
}

328
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt
View file

@ -1,168 +1,168 @@
package dev.adamko.kafkatorio.processor.tileserver package dev.adamko.kafkatorio.processor.tileserver
import com.github.palindromicity.syslog.AllowableDeviations //import com.github.palindromicity.syslog.AllowableDeviations
import com.github.palindromicity.syslog.SyslogParser //import com.github.palindromicity.syslog.SyslogParser
import com.github.palindromicity.syslog.SyslogParserBuilder //import com.github.palindromicity.syslog.SyslogParserBuilder
import com.github.palindromicity.syslog.SyslogSpecification //import com.github.palindromicity.syslog.SyslogSpecification
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys //import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
import dev.adamko.kafkatorio.processor.config.ApplicationProperties //import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.tileserver.SyslogMsg.Companion.toSyslogMsg //import dev.adamko.kafkatorio.processor.tileserver.SyslogMsg.Companion.toSyslogMsg
import io.ktor.network.selector.SelectorManager //import io.ktor.network.selector.SelectorManager
import io.ktor.network.sockets.aSocket //import io.ktor.network.sockets.aSocket
import io.ktor.network.sockets.awaitClosed //import io.ktor.network.sockets.awaitClosed
import io.ktor.network.sockets.openReadChannel //import io.ktor.network.sockets.openReadChannel
import io.ktor.utils.io.readUTF8Line //import io.ktor.utils.io.readUTF8Line
import java.util.EnumSet //import java.util.EnumSet
import kotlinx.coroutines.CoroutineDispatcher //import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers //import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope //import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.MutableSharedFlow //import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.SharedFlow //import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.asSharedFlow //import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.isActive //import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch //import kotlinx.coroutines.launch
import kotlinx.coroutines.yield //import kotlinx.coroutines.yield
class SyslogSocketServer(
appProps: ApplicationProperties,
dispatcher: CoroutineDispatcher = Dispatchers.IO,
) {
private val selectorManager: SelectorManager = SelectorManager(dispatcher)
private val serverSocketTcp = aSocket(selectorManager)
.tcp()
.bind(
appProps.socketServerHost,
appProps.socketServerPort,
)
// private val serverSocketUdp: BoundDatagramSocket = aSocket(selectorManager)
// .udp()
// .bind(
// localAddress = InetSocketAddress(
// appProps.socketServerHost,
// appProps.socketServerPortUdp,
// ),
// )
private val syslogParser: SyslogParser = SyslogParserBuilder()
.withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION))
.forSpecification(SyslogSpecification.RFC_5424)
.build()
private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
val messages: SharedFlow<SyslogMsg>
get() = _messages.asSharedFlow()
suspend fun start() {
// udp()
tcp()
}
// private suspend fun udp() {
// println("[SyslogSocketServer] started listening: ${serverSocketUdp.localAddress}")
// //
// serverSocketUdp.incoming //
// .receiveAsFlow() //class SyslogSocketServer(
// .map {udpPacket: Datagram -> // appProps: ApplicationProperties,
// val msg = udpPacket.packet.readText() // dispatcher: CoroutineDispatcher = Dispatchers.IO,
// println("[SyslogSocketServer] received msg: $msg") //) {
// syslogParser.parseLine(msg).toSyslogMsg() //
// }.collect(_messages) // private val selectorManager: SelectorManager = SelectorManager(dispatcher)
//
//
// private val serverSocketTcp = aSocket(selectorManager)
// .tcp()
// .bind(
// appProps.socketServerHost,
// appProps.socketServerPort,
// )
//
//
//// private val serverSocketUdp: BoundDatagramSocket = aSocket(selectorManager)
//// .udp()
//// .bind(
//// localAddress = InetSocketAddress(
//// appProps.socketServerHost,
//// appProps.socketServerPortUdp,
//// ),
//// )
//
//
// private val syslogParser: SyslogParser = SyslogParserBuilder()
// .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION))
// .forSpecification(SyslogSpecification.RFC_5424)
// .build()
//
//
// private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
// val messages: SharedFlow<SyslogMsg>
// get() = _messages.asSharedFlow()
//
//
// suspend fun start() {
//// udp()
// tcp()
// } // }
//
private suspend fun tcp() = coroutineScope { //// private suspend fun udp() {
println("[SyslogSocketServer] started listening: ${serverSocketTcp.localAddress}") //// println("[SyslogSocketServer] started listening: ${serverSocketUdp.localAddress}")
////
while (isActive) { //// serverSocketUdp.incoming
val socket = serverSocketTcp.accept() //// .receiveAsFlow()
//// .map {udpPacket: Datagram ->
launch { //// val msg = udpPacket.packet.readText()
val channel = socket.openReadChannel() //// println("[SyslogSocketServer] received msg: $msg")
try { //// syslogParser.parseLine(msg).toSyslogMsg()
while (socket.socketContext.isActive) { //// }.collect(_messages)
val line = channel.readUTF8Line() //// }
println("[SyslogSocketServer] received: $line") //
// private suspend fun tcp() = coroutineScope {
val syslog = syslogParser.parseLine(line).toSyslogMsg() // println("[SyslogSocketServer] started listening: ${serverSocketTcp.localAddress}")
//
_messages.emit(syslog) // while (isActive) {
// val socket = serverSocketTcp.accept()
yield() //
} // launch {
} catch (e: Exception) { // val channel = socket.openReadChannel()
socket.awaitClosed() // try {
} // while (socket.socketContext.isActive) {
} // val line = channel.readUTF8Line()
} // println("[SyslogSocketServer] received: $line")
//
yield() // val syslog = syslogParser.parseLine(line).toSyslogMsg()
} //
} // _messages.emit(syslog)
//
// yield()
data class SyslogMsg( // }
val src: Map<String, String?> // } catch (e: Exception) {
) { // socket.awaitClosed()
// }
val message: String? // }
get() = src[SyslogFieldKeys.MESSAGE.field] // }
//
val headerAppName: String? // yield()
get() = src[SyslogFieldKeys.HEADER_APPNAME.field] // }
//}
val headerHostName: String? //
get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field] //
//data class SyslogMsg(
val headerPri: String? // val src: Map<String, String?>
get() = src[SyslogFieldKeys.HEADER_PRI.field] //) {
//
val headerPriSeverity: String? // val message: String?
get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field] // get() = src[SyslogFieldKeys.MESSAGE.field]
//
val headerPriFacility: String? // val headerAppName: String?
get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field] // get() = src[SyslogFieldKeys.HEADER_APPNAME.field]
//
val headerProcId: String? // val headerHostName: String?
get() = src[SyslogFieldKeys.HEADER_PROCID.field] // get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field]
//
val headerTimestamp: String? // val headerPri: String?
get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field] // get() = src[SyslogFieldKeys.HEADER_PRI.field]
//
val headerMsgId: String? // val headerPriSeverity: String?
get() = src[SyslogFieldKeys.HEADER_MSGID.field] // get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field]
//
val headerVersion: String? // val headerPriFacility: String?
get() = src[SyslogFieldKeys.HEADER_VERSION.field] // get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field]
//
val structuredBase: String? // val headerProcId: String?
get() = src[SyslogFieldKeys.STRUCTURED_BASE.field] // get() = src[SyslogFieldKeys.HEADER_PROCID.field]
//
val structuredElementIdFmt: String? // val headerTimestamp: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field] // get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field]
//
val structuredElementIdPnameFmt: String? // val headerMsgId: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field] // get() = src[SyslogFieldKeys.HEADER_MSGID.field]
//
val structuredElementIdPnamePattern: String? // val headerVersion: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field] // get() = src[SyslogFieldKeys.HEADER_VERSION.field]
//
companion object { // val structuredBase: String?
fun Map<String, Any?>.toSyslogMsg(): SyslogMsg { // get() = src[SyslogFieldKeys.STRUCTURED_BASE.field]
val map = entries.mapNotNull { (k, v) -> //
when (v) { // val structuredElementIdFmt: String?
is String -> k to v // get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field]
else -> null //
} // val structuredElementIdPnameFmt: String?
}.toMap() // get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field]
return SyslogMsg(map) //
} // val structuredElementIdPnamePattern: String?
} // get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field]
} //
// companion object {
// fun Map<String, Any?>.toSyslogMsg(): SyslogMsg {
// val map = entries.mapNotNull { (k, v) ->
// when (v) {
// is String -> k to v
// else -> null
// }
// }.toMap()
// return SyslogMsg(map)
// }
// }
//}

168
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebMapTileServer.kt
View file

@ -1,86 +1,86 @@
package dev.adamko.kafkatorio.processor.tileserver package dev.adamko.kafkatorio.processor.tileserver
import dev.adamko.kafkatorio.processor.config.ApplicationProperties //import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import java.security.MessageDigest //import java.security.MessageDigest
import org.http4k.core.Filter //import org.http4k.core.Filter
import org.http4k.core.HttpHandler //import org.http4k.core.HttpHandler
import org.http4k.core.MemoryBody //import org.http4k.core.MemoryBody
import org.http4k.core.Method.GET //import org.http4k.core.Method.GET
import org.http4k.core.Request //import org.http4k.core.Request
import org.http4k.core.Response //import org.http4k.core.Response
import org.http4k.core.Status //import org.http4k.core.Status
import org.http4k.core.then //import org.http4k.core.then
import org.http4k.routing.ResourceLoader //import org.http4k.routing.ResourceLoader
import org.http4k.routing.RoutingHttpHandler //import org.http4k.routing.RoutingHttpHandler
import org.http4k.routing.bind //import org.http4k.routing.bind
import org.http4k.routing.routes //import org.http4k.routing.routes
import org.http4k.routing.static //import org.http4k.routing.static
//
internal class WebMapTileServer( //internal class WebMapTileServer(
appProps: ApplicationProperties // appProps: ApplicationProperties
) { //) {
//
private val tilesLoader = ResourceLoader.Directory(appProps.serverDataDir.toFile().canonicalPath) // private val tilesLoader = ResourceLoader.Directory(appProps.serverDataDir.toFile().canonicalPath)
//
//
private val filters = listOf( // private val filters = listOf(
ETagChecker(), // ETagChecker(),
).reduce(Filter::then) // ).reduce(Filter::then)
//
//
private val routes: RoutingHttpHandler = routes( // private val routes: RoutingHttpHandler = routes(
"/tiles" bind GET to static(tilesLoader).withFilter(filters) // "/tiles" bind GET to static(tilesLoader).withFilter(filters)
) // )
//
//
fun build(): HttpHandler { // fun build(): HttpHandler {
return routes // return routes
} // }
//
//
private class ETagChecker : Filter { // private class ETagChecker : Filter {
//
override fun invoke(next: HttpHandler): HttpHandler = { request: Request -> // override fun invoke(next: HttpHandler): HttpHandler = { request: Request ->
val requestedETag = request.header("If-None-Match") // val requestedETag = request.header("If-None-Match")
//
var response = next(request) // var response = next(request)
val responseBytes = response.body.stream.use { it.readAllBytes() } // val responseBytes = response.body.stream.use { it.readAllBytes() }
val currentETag = responseBytes.md5() // val currentETag = responseBytes.md5()
//
response = response.body(MemoryBody(responseBytes)) // response = response.body(MemoryBody(responseBytes))
//
when { // when {
currentETag == null -> { // currentETag == null -> {
// println("[${request.etagKey}] currentETag is null (requested: $requestedETag), executing request") //// println("[${request.etagKey}] currentETag is null (requested: $requestedETag), executing request")
response // response
.header("Cache-Control", "no-cache, max-age=0, must-revalidate") // .header("Cache-Control", "no-cache, max-age=0, must-revalidate")
} // }
requestedETag == currentETag -> { // requestedETag == currentETag -> {
// println("[${request.etagKey}] currentETag == requested, returning 304 $requestedETag") //// println("[${request.etagKey}] currentETag == requested, returning 304 $requestedETag")
Response(Status.NOT_MODIFIED) // Response(Status.NOT_MODIFIED)
.header("Cache-Control", "no-cache, max-age=0, must-revalidate") // .header("Cache-Control", "no-cache, max-age=0, must-revalidate")
} // }
else -> { // else -> {
// println("[${request.etagKey}] currentETag != requested, executing request and adding currentETag:$currentETag (requested:$requestedETag)") //// println("[${request.etagKey}] currentETag != requested, executing request and adding currentETag:$currentETag (requested:$requestedETag)")
response // response
.header("ETag", currentETag) // .header("ETag", currentETag)
// .header("Cache-Control", "no-store") //// .header("Cache-Control", "no-store")
.header("Cache-Control", "no-cache, max-age=0, must-revalidate") // .header("Cache-Control", "no-cache, max-age=0, must-revalidate")
} // }
} // }
} // }
} // }
//
//
companion object { // companion object {
private fun ByteArray.md5(): String? { // private fun ByteArray.md5(): String? {
return if (isEmpty()) { // return if (isEmpty()) {
null // null
} else { // } else {
MessageDigest.getInstance("MD5") // MessageDigest.getInstance("MD5")
.digest(this) // .digest(this)
.joinToString("") { "%02x".format(it) } // .joinToString("") { "%02x".format(it) }
} // }
} // }
} // }
} //}

30
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebServer.kt
View file

@ -1,16 +1,16 @@
package dev.adamko.kafkatorio.processor.tileserver package dev.adamko.kafkatorio.processor.tileserver
//
import dev.adamko.kafkatorio.processor.config.ApplicationProperties //import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import org.http4k.server.Http4kServer //import org.http4k.server.Http4kServer
import org.http4k.server.PolyHandler //import org.http4k.server.PolyHandler
import org.http4k.server.Undertow //import org.http4k.server.Undertow
import org.http4k.server.asServer //import org.http4k.server.asServer
//
internal fun webServer( //internal fun webServer(
tileServer: WebMapTileServer, // tileServer: WebMapTileServer,
wsServer: WebsocketServer, // wsServer: WebsocketServer,
appProps: ApplicationProperties, // appProps: ApplicationProperties,
): Http4kServer = PolyHandler( //): Http4kServer = PolyHandler(
tileServer.build(), // tileServer.build(),
wsServer.build(), // wsServer.build(),
).asServer(Undertow(appProps.webPort())) //).asServer(Undertow(appProps.webPort()))

280
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/WebsocketServer.kt
View file

@ -1,141 +1,141 @@
package dev.adamko.kafkatorio.processor.tileserver package dev.adamko.kafkatorio.processor.tileserver
//
import kotlin.coroutines.resume //import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException //import kotlin.coroutines.resumeWithException
import kotlin.time.Duration.Companion.seconds //import kotlin.time.Duration.Companion.seconds
import kotlin.time.TimeSource //import kotlin.time.TimeSource
import kotlinx.coroutines.CoroutineExceptionHandler //import kotlinx.coroutines.CoroutineExceptionHandler
import kotlinx.coroutines.CoroutineName //import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope //import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers //import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob //import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.channels.BufferOverflow //import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.delay //import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableSharedFlow //import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.launchIn //import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach //import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.withIndex //import kotlinx.coroutines.flow.withIndex
import kotlinx.coroutines.isActive //import kotlinx.coroutines.isActive
import kotlinx.coroutines.job //import kotlinx.coroutines.job
import kotlinx.coroutines.launch //import kotlinx.coroutines.launch
import kotlinx.coroutines.supervisorScope //import kotlinx.coroutines.supervisorScope
import kotlinx.coroutines.suspendCancellableCoroutine //import kotlinx.coroutines.suspendCancellableCoroutine
import org.http4k.routing.bind //import org.http4k.routing.bind
import org.http4k.routing.websockets //import org.http4k.routing.websockets
import org.http4k.websocket.Websocket //import org.http4k.websocket.Websocket
import org.http4k.websocket.WsHandler //import org.http4k.websocket.WsHandler
import org.http4k.websocket.WsMessage //import org.http4k.websocket.WsMessage
import org.http4k.websocket.WsStatus //import org.http4k.websocket.WsStatus
//
//
class WebsocketServer { //class WebsocketServer {
//
private val serverScope = CoroutineScope( // private val serverScope = CoroutineScope(
CoroutineName("WsServer") + // CoroutineName("WsServer") +
Dispatchers.IO + // Dispatchers.IO +
SupervisorJob() + // SupervisorJob() +
CoroutineExceptionHandler { _, e -> // CoroutineExceptionHandler { _, e ->
println("[ws.WsServer] handling exception ${e.message}") // println("[ws.WsServer] handling exception ${e.message}")
} // }
) // )
//
//
private val clients: MutableSharedFlow<Websocket> = MutableSharedFlow( // private val clients: MutableSharedFlow<Websocket> = MutableSharedFlow(
replay = 100, // replay = 100,
extraBufferCapacity = 100, // extraBufferCapacity = 100,
BufferOverflow.DROP_OLDEST, // BufferOverflow.DROP_OLDEST,
) // )
//
//
private val messages: MutableSharedFlow<String> = MutableSharedFlow( // private val messages: MutableSharedFlow<String> = MutableSharedFlow(
replay = 100, // replay = 100,
extraBufferCapacity = 100, // extraBufferCapacity = 100,
BufferOverflow.DROP_OLDEST, // BufferOverflow.DROP_OLDEST,
) // )
//
//
private fun bindClient(ws: Websocket) = serverScope.launch { // private fun bindClient(ws: Websocket) = serverScope.launch {
println("[ws.bindClient] ${ws.upgradeRequest.toMessage()}") // println("[ws.bindClient] ${ws.upgradeRequest.toMessage()}")
clients.emit(ws) // clients.emit(ws)
} // }
//
//
fun sendMessage(msg: String) = serverScope.launch { // fun sendMessage(msg: String) = serverScope.launch {
println("[ws.sendMessage] $msg") // println("[ws.sendMessage] $msg")
messages.emit(msg) // messages.emit(msg)
} // }
//
//
fun build(): WsHandler { // fun build(): WsHandler {
val startTime = TimeSource.Monotonic.markNow() // val startTime = TimeSource.Monotonic.markNow()
//
serverScope.launch { // serverScope.launch {
supervisorScope { // supervisorScope {
clients // clients
.withIndex() // .withIndex()
.onEach { (id, ws) -> // .onEach { (id, ws) ->
createClient(id, ws) // createClient(id, ws)
}.launchIn(serverScope) // }.launchIn(serverScope)
} // }
} // }
//
serverScope.launch { // serverScope.launch {
while (isActive) { // while (isActive) {
delay(10.seconds) // delay(10.seconds)
sendMessage("elapsed ${startTime.elapsedNow()}") // sendMessage("elapsed ${startTime.elapsedNow()}")
} // }
} // }
//
return websockets( // return websockets(
"/ws/foo" bind { ws: Websocket -> // "/ws/foo" bind { ws: Websocket ->
serverScope.launch { bindClient(ws) } // serverScope.launch { bindClient(ws) }
} // }
) // )
} // }
//
//
private fun createClient(id: Int, ws: Websocket) { // private fun createClient(id: Int, ws: Websocket) {
val clientJob = SupervisorJob(serverScope.coroutineContext.job) // val clientJob = SupervisorJob(serverScope.coroutineContext.job)
//
serverScope.launch(CoroutineName("ws-client-$id") + clientJob) { // serverScope.launch(CoroutineName("ws-client-$id") + clientJob) {
launch { // launch {
suspendCancellableCoroutine<WsStatus> { cont -> // suspendCancellableCoroutine<WsStatus> { cont ->
println("[ws-client] entering suspendCancellableCoroutine") // println("[ws-client] entering suspendCancellableCoroutine")
//
cont.invokeOnCancellation { e: Throwable? -> // cont.invokeOnCancellation { e: Throwable? ->
launch { sendMessage("[ws-client.invokeOnCompletion] closing client ws ${e?.message}") } // launch { sendMessage("[ws-client.invokeOnCompletion] closing client ws ${e?.message}") }
when (e) { // when (e) {
null -> ws.close(WsStatus.NORMAL) // null -> ws.close(WsStatus.NORMAL)
else -> ws.close(WsStatus.ABNORMAL_CLOSE) // else -> ws.close(WsStatus.ABNORMAL_CLOSE)
} // }
} // }
//
ws.onError { e: Throwable -> // ws.onError { e: Throwable ->
launch { sendMessage("[ws-client.onError] client $id error [${e.message}]") } // launch { sendMessage("[ws-client.onError] client $id error [${e.message}]") }
if (!cont.isCompleted) { // if (!cont.isCompleted) {
cont.resumeWithException(e) // cont.resumeWithException(e)
} // }
} // }
//
ws.onClose { status: WsStatus -> // ws.onClose { status: WsStatus ->
launch { sendMessage("[ws-client.onClose] client $id disconnected [${status}]") } // launch { sendMessage("[ws-client.onClose] client $id disconnected [${status}]") }
if (!cont.isCompleted) { // if (!cont.isCompleted) {
cont.resume(status) // cont.resume(status)
} // }
} // }
//
ws.onMessage { wsMsg -> // ws.onMessage { wsMsg ->
launch { sendMessage("$id says: ${wsMsg.bodyString()}") } // launch { sendMessage("$id says: ${wsMsg.bodyString()}") }
} // }
} // }
} // }
//
println("[ws-client] $id listening to messages") // println("[ws-client] $id listening to messages")
messages // messages
.onEach { msg -> // .onEach { msg ->
println("[ws-client] sending message to $id") // println("[ws-client] sending message to $id")
ws.send(WsMessage(msg)) // ws.send(WsMessage(msg))
}.launchIn(this) // }.launchIn(this)
} // }
} // }
} //}

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/config/ApplicationProperties.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.processor.config package dev.adamko.kafkatorio.server.config
import com.sksamuel.hoplite.ConfigLoaderBuilder import com.sksamuel.hoplite.ConfigLoaderBuilder
import com.sksamuel.hoplite.addEnvironmentSource import com.sksamuel.hoplite.addEnvironmentSource

6
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/PortVal.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/config/PortVal.kt
View file

@ -1,9 +1,7 @@
package dev.adamko.kafkatorio.processor.config package dev.adamko.kafkatorio.server.config
import org.http4k.cloudnative.env.Port
/** Improved (slightly) version of http4k's [Port]. */ /** Improved (slightly) version of http4k's `Port`. */
@JvmInline @JvmInline
value class PortVal(val value: Int) { value class PortVal(val value: Int) {

5
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes/kxsJson.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/config/kxsJson.kt
View file

@ -1,8 +1,9 @@
package dev.adamko.kafkatorio.processor.serdes package dev.adamko.kafkatorio.server.config
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
val jsonMapper: Json = Json { val jsonMapper: Json = Json {
prettyPrint = true prettyPrint = true
prettyPrintIndent = " " prettyPrintIndent = " "

32
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/eventsServer.kt Normal file
View file

@ -0,0 +1,32 @@
package dev.adamko.kafkatorio.server
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
import dev.adamko.kafkatorio.server.web.startWebServer
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
suspend fun main(): Unit = coroutineScope {
val appProps = ApplicationProperties.load()
val websocketServer = WebmapWebsocketServer()
val syslogSocketServer = SyslogSocketServer(appProps)
launch {
startWebServer(appProps, websocketServer)
}
launch {
syslogSocketServer.start()
}
// launch(Dispatchers.Default.limitedParallelism(2)) {
// eventsProcessor(appProps, syslogSocketServer, websocketServer)
// }
}

38
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/kafkatorioEventsServer.kt
View file

@ -1,38 +0,0 @@
package dev.adamko.kafkatorio.server
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import io.ktor.http.HttpStatusCode
import io.ktor.server.application.Application
import io.ktor.server.application.install
import io.ktor.server.cio.CIO
import io.ktor.server.engine.embeddedServer
import io.ktor.server.plugins.callloging.CallLogging
import io.ktor.server.plugins.statuspages.StatusPages
import io.ktor.server.response.respondText
fun main() {
val appProps = ApplicationProperties.load()
embeddedServer(CIO, port = appProps.webPort.value) {
common()
webmapWebsocketServer()
webmapTileServer(appProps)
syslogSocketServer(appProps)
}.start(wait = true)
}
fun Application.common() {
install(StatusPages) {
exception<Throwable> { call, cause ->
call.respondText(text = "500: $cause", status = HttpStatusCode.InternalServerError)
}
}
install(CallLogging)
}

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessor.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/DebounceProcessor.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.processor.misc package dev.adamko.kafkatorio.server.processor
import dev.adamko.kotka.extensions.component1 import dev.adamko.kotka.extensions.component1
import dev.adamko.kotka.extensions.component2 import dev.adamko.kotka.extensions.component2

21
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/KafkatorioTopology.kt
View file

@ -1,15 +1,12 @@
package dev.adamko.kafkatorio.processor package dev.adamko.kafkatorio.server.processor
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.config.ApplicationProperties import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer import dev.adamko.kafkatorio.server.processor.topology.*
import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
import dev.adamko.kafkatorio.processor.topology.broadcastToWebsocket import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import dev.adamko.kafkatorio.processor.topology.colourMapChunks import dev.adamko.kafkatorio.server.web.websocket.broadcastToWebsocket
import dev.adamko.kafkatorio.processor.topology.factorioServerPacketStream
import dev.adamko.kafkatorio.processor.topology.saveMapTiles
import dev.adamko.kafkatorio.processor.topology.splitFactorioServerPacketStream
import java.util.Properties import java.util.Properties
import kotlin.coroutines.resume import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException import kotlin.coroutines.resumeWithException
@ -40,7 +37,7 @@ import org.apache.kafka.streams.TopologyDescription
internal class KafkatorioTopology( internal class KafkatorioTopology(
private val appProps: ApplicationProperties, private val appProps: ApplicationProperties,
private val websocketServer: WebsocketServer, private val wsServer: WebmapWebsocketServer,
private val syslogServer: SyslogSocketServer, private val syslogServer: SyslogSocketServer,
) { ) {
@ -86,7 +83,7 @@ internal class KafkatorioTopology(
private suspend fun websocketBroadcaster() { private suspend fun websocketBroadcaster() {
val builder = StreamsBuilder() val builder = StreamsBuilder()
broadcastToWebsocket(websocketServer, builder) broadcastToWebsocket(wsServer, builder)
val topology = builder.build() val topology = builder.build()

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/StreamsExceptionHandler.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/StreamsExceptionHandler.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.processor package dev.adamko.kafkatorio.server.processor
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse

5
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/KafkatorioKafkaAdmin.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/admin/KafkatorioKafkaAdmin.kt
View file

@ -1,6 +1,7 @@
package dev.adamko.kafkatorio.processor.admin package dev.adamko.kafkatorio.server.processor.admin
import dev.adamko.kafkatorio.processor.config.ApplicationProperties import dev.adamko.kafkatorio.server.processor.allTopics
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import mu.KotlinLogging import mu.KotlinLogging
import org.apache.kafka.clients.admin.Admin import org.apache.kafka.clients.admin.Admin

37
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/eventProcessor.kt Normal file
View file

@ -0,0 +1,37 @@
package dev.adamko.kafkatorio.server.processor
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
import dev.adamko.kafkatorio.server.processor.admin.KafkatorioKafkaAdmin
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import kotlinx.coroutines.awaitCancellation
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
suspend fun eventsProcessor(
appProps: ApplicationProperties,
syslogServer: SyslogSocketServer,
wsServer: WebmapWebsocketServer,
): Unit = coroutineScope {
val admin = KafkatorioKafkaAdmin(appProps)
admin.createKafkatorioTopics()
// val syslogServer = SyslogSocketServer(appProps)
// val wsServer = WebsocketServer()
// val tileServer = WebMapTileServer(appProps)
val topology = KafkatorioTopology(appProps, wsServer, syslogServer)
launch {
topology.start()
println("launched KafkatorioTopology")
awaitCancellation()
}
}

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/kafkatorioTopics.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.processor.admin package dev.adamko.kafkatorio.server.processor
import dev.adamko.kafkatorio.schema.packets.ConfigurationUpdate import dev.adamko.kafkatorio.schema.packets.ConfigurationUpdate
import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate

10
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/colourMapChunks.kt
View file

@ -1,9 +1,9 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.processor.admin.TOPIC_MAP_CHUNK_COLOURED_032_STATE import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_COLOURED_032_STATE
import dev.adamko.kafkatorio.processor.admin.TOPIC_MAP_CHUNK_COLOURED_STATE import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_COLOURED_STATE
import dev.adamko.kafkatorio.processor.misc.DebounceProcessor.Companion.addDebounceProcessor import dev.adamko.kafkatorio.server.processor.DebounceProcessor.Companion.addDebounceProcessor
import dev.adamko.kafkatorio.processor.serdes.kxsBinary import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.schema.common.ChunkSize import dev.adamko.kafkatorio.schema.common.ChunkSize
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId

9
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/factorioServerPackets.kt
View file

@ -1,9 +1,8 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.server.processor.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG import dev.adamko.kafkatorio.server.processor.topicName
import dev.adamko.kafkatorio.processor.admin.topicName import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.Tick import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/model.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/model.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.processor.topology
import com.sksamuel.scrimage.color.RGBColor import com.sksamuel.scrimage.color.RGBColor
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex

6
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/packetDataStreams.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/packetDataStreams.kt
View file

@ -1,7 +1,7 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.processor.admin.topicName import dev.adamko.kafkatorio.server.processor.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData

4
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/prototypes.kt
View file

@ -1,6 +1,6 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.processor.serdes.kxsBinary import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioPrototype import dev.adamko.kafkatorio.schema.common.FactorioPrototype
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId

10
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/saveMapTiles.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/saveMapTiles.kt
View file

@ -1,12 +1,12 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.processor.topology
import com.sksamuel.scrimage.ImmutableImage import com.sksamuel.scrimage.ImmutableImage
import com.sksamuel.scrimage.ScaleMethod import com.sksamuel.scrimage.ScaleMethod
import com.sksamuel.scrimage.nio.PngWriter import com.sksamuel.scrimage.nio.PngWriter
import dev.adamko.kafkatorio.processor.admin.TOPIC_BROADCAST_TO_WEBSOCKET import dev.adamko.kafkatorio.server.processor.TOPIC_BROADCAST_TO_WEBSOCKET
import dev.adamko.kafkatorio.processor.admin.TOPIC_MAP_CHUNK_COLOURED_STATE import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_COLOURED_STATE
import dev.adamko.kafkatorio.processor.serdes.jsonMapper import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.processor.serdes.kxsBinary import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId import dev.adamko.kafkatorio.schema.common.ServerMapChunkId

15
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/websocketBroadcast.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/websocketBroadcaster.kt
View file

@ -1,10 +1,9 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.web.websocket
import dev.adamko.kafkatorio.processor.admin.TOPIC_BROADCAST_TO_WEBSOCKET import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
import dev.adamko.kafkatorio.processor.admin.topicName import dev.adamko.kafkatorio.server.processor.TOPIC_BROADCAST_TO_WEBSOCKET
import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer import dev.adamko.kafkatorio.server.processor.topicName
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import dev.adamko.kafkatorio.schema.packets.EventServerPacket import dev.adamko.kafkatorio.schema.packets.EventServerPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
@ -32,8 +31,8 @@ private const val pid = "websocketBroadcast"
fun broadcastToWebsocket( fun broadcastToWebsocket(
websocketServer: WebsocketServer, websocketServer: WebmapWebsocketServer,
builder: StreamsBuilder = StreamsBuilder(), builder: StreamsBuilder,
) { ) {
builder.broadcastKafkatorioPackets() builder.broadcastKafkatorioPackets()
@ -45,7 +44,7 @@ fun broadcastToWebsocket(
"$pid.websocket-server.send-message" "$pid.websocket-server.send-message"
) { _, esPacket -> ) { _, esPacket ->
runBlocking { runBlocking {
websocketServer.sendMessage(jsonMapper.encodeToString(esPacket)) websocketServer.sendMessageToClients(jsonMapper.encodeToString(esPacket))
} }
} }
} }

63
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/socket/SyslogMsg.kt Normal file
View file

@ -0,0 +1,63 @@
package dev.adamko.kafkatorio.server.socket
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
data class SyslogMsg(
val src: Map<String, String?>
) {
val message: String?
get() = src[SyslogFieldKeys.MESSAGE.field]
val headerAppName: String?
get() = src[SyslogFieldKeys.HEADER_APPNAME.field]
val headerHostName: String?
get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field]
val headerPri: String?
get() = src[SyslogFieldKeys.HEADER_PRI.field]
val headerPriSeverity: String?
get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field]
val headerPriFacility: String?
get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field]
val headerProcId: String?
get() = src[SyslogFieldKeys.HEADER_PROCID.field]
val headerTimestamp: String?
get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field]
val headerMsgId: String?
get() = src[SyslogFieldKeys.HEADER_MSGID.field]
val headerVersion: String?
get() = src[SyslogFieldKeys.HEADER_VERSION.field]
val structuredBase: String?
get() = src[SyslogFieldKeys.STRUCTURED_BASE.field]
val structuredElementIdFmt: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field]
val structuredElementIdPnameFmt: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field]
val structuredElementIdPnamePattern: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field]
companion object {
fun Map<String, Any?>.toSyslogMsg(): SyslogMsg {
val map = entries.mapNotNull { (k, v) ->
when (v) {
is String -> k to v
else -> null
}
}.toMap()
return SyslogMsg(map)
}
}
}

99
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/socket/SyslogSocketServer.kt Normal file
View file

@ -0,0 +1,99 @@
package dev.adamko.kafkatorio.server.socket
import com.github.palindromicity.syslog.AllowableDeviations
import com.github.palindromicity.syslog.SyslogParser
import com.github.palindromicity.syslog.SyslogParserBuilder
import com.github.palindromicity.syslog.SyslogSpecification
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.socket.SyslogMsg.Companion.toSyslogMsg
import io.ktor.network.selector.SelectorManager
import io.ktor.network.sockets.Socket
import io.ktor.network.sockets.aSocket
import io.ktor.network.sockets.openReadChannel
import io.ktor.utils.io.readUTF8Line
import java.util.EnumSet
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext
import kotlinx.coroutines.yield
class SyslogSocketServer(
appProps: ApplicationProperties,
) {
private val selectorManager: SelectorManager = SelectorManager(Dispatchers.IO)
private val serverSocketTcp = aSocket(selectorManager)
.tcp()
.bind(
appProps.socketServerHost,
appProps.socketServerPort,
)
private val syslogParser: SyslogParser = SyslogParserBuilder()
.withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION))
.forSpecification(SyslogSpecification.RFC_5424)
.build()
private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
val messages: SharedFlow<SyslogMsg>
get() = _messages.asSharedFlow()
suspend fun start(): Unit = coroutineScope {
log("started listening: ${serverSocketTcp.localAddress}")
while (isActive) {
val socket: Socket = serverSocketTcp.accept()
log("new socket ${socket.description()}")
launch {
val channel = socket.openReadChannel()
try {
while (socket.socketContext.isActive) {
val line = runCatching {
channel.readUTF8Line()
}.fold(
onSuccess = { line ->
log("received '$line' from $socket")
line
},
onFailure = { e ->
log("reading line from socket failed ${socket.description()}, cause: $e")
null
},
)
if (line != null) {
val syslog = syslogParser.parseLine(line).toSyslogMsg()
_messages.emit(syslog)
}
yield()
}
} catch (e: Exception) {
log("closing socket ${socket.description()}, cause: $e")
withContext(Dispatchers.IO) { socket.close() }
}
}
yield()
}
}
companion object {
private fun log(msg: String) = println("[SyslogSocketServer] $msg")
private fun Socket.description(): String = "Socket[local:$localAddress, remote:$remoteAddress]"
}
}

152
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/syslogSocketServer.kt
View file

@ -1,152 +0,0 @@
package dev.adamko.kafkatorio.server
import com.github.palindromicity.syslog.AllowableDeviations
import com.github.palindromicity.syslog.SyslogParser
import com.github.palindromicity.syslog.SyslogParserBuilder
import com.github.palindromicity.syslog.SyslogSpecification
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.server.SyslogMsg.Companion.toSyslogMsg
import io.ktor.network.selector.SelectorManager
import io.ktor.network.sockets.aSocket
import io.ktor.network.sockets.awaitClosed
import io.ktor.network.sockets.openReadChannel
import io.ktor.server.application.Application
import io.ktor.utils.io.readUTF8Line
import java.util.EnumSet
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.yield
fun Application.syslogSocketServer(appProps: ApplicationProperties) {
val server = SyslogSocketServer(appProps)
launch {
server.start()
}
}
class SyslogSocketServer(
appProps: ApplicationProperties,
) {
private val selectorManager: SelectorManager = SelectorManager(Dispatchers.IO)
private val serverSocketTcp = aSocket(selectorManager)
.tcp()
.bind(
appProps.socketServerHost,
appProps.socketServerPort,
)
private val syslogParser: SyslogParser = SyslogParserBuilder()
.withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION))
.forSpecification(SyslogSpecification.RFC_5424)
.build()
private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
val messages: SharedFlow<SyslogMsg>
get() = _messages.asSharedFlow()
suspend fun start(): Unit = coroutineScope {
println("[SyslogSocketServer] started listening: ${serverSocketTcp.localAddress}")
while (isActive) {
val socket = serverSocketTcp.accept()
launch {
val channel = socket.openReadChannel()
try {
while (socket.socketContext.isActive) {
val line = runCatching { channel.readUTF8Line() }.getOrNull()
if (line != null) {
println("[SyslogSocketServer] received: $line")
val syslog = syslogParser.parseLine(line).toSyslogMsg()
_messages.emit(syslog)
}
yield()
}
} catch (e: Exception) {
socket.awaitClosed()
}
}
yield()
}
}
}
data class SyslogMsg(
val src: Map<String, String?>
) {
val message: String?
get() = src[SyslogFieldKeys.MESSAGE.field]
val headerAppName: String?
get() = src[SyslogFieldKeys.HEADER_APPNAME.field]
val headerHostName: String?
get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field]
val headerPri: String?
get() = src[SyslogFieldKeys.HEADER_PRI.field]
val headerPriSeverity: String?
get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field]
val headerPriFacility: String?
get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field]
val headerProcId: String?
get() = src[SyslogFieldKeys.HEADER_PROCID.field]
val headerTimestamp: String?
get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field]
val headerMsgId: String?
get() = src[SyslogFieldKeys.HEADER_MSGID.field]
val headerVersion: String?
get() = src[SyslogFieldKeys.HEADER_VERSION.field]
val structuredBase: String?
get() = src[SyslogFieldKeys.STRUCTURED_BASE.field]
val structuredElementIdFmt: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field]
val structuredElementIdPnameFmt: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field]
val structuredElementIdPnamePattern: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field]
companion object {
fun Map<String, Any?>.toSyslogMsg(): SyslogMsg {
val map = entries.mapNotNull { (k, v) ->
when (v) {
is String -> k to v
else -> null
}
}.toMap()
return SyslogMsg(map)
}
}
}

121
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/factorioServersRoutes.kt Normal file
View file

@ -0,0 +1,121 @@
package dev.adamko.kafkatorio.server.web.rest
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.TilePngFilename
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import io.ktor.http.HttpStatusCode
import io.ktor.http.content.EntityTagVersion
import io.ktor.http.content.versions
import io.ktor.resources.Resource
import io.ktor.server.application.ApplicationCall
import io.ktor.server.application.application
import io.ktor.server.application.call
import io.ktor.server.application.log
import io.ktor.server.http.content.LastModifiedVersion
import io.ktor.server.resources.get
import io.ktor.server.response.respond
import io.ktor.server.response.respondFile
import io.ktor.server.routing.Routing
import io.ktor.util.combineSafe
import io.ktor.util.pipeline.PipelineContext
import kotlin.io.path.Path
import kotlinx.serialization.Serializable
fun Routing.factorioServersRoutes(
appProps: ApplicationProperties,
) {
get<FactorioServers> {
call.respond(HttpStatusCode.NotImplemented)
}
get<FactorioServer.MapData.Tile> { tile ->
handleTileRequest(tile, appProps)
}
}
private suspend fun PipelineContext<Unit, ApplicationCall>.handleTileRequest(
tile: FactorioServer.MapData.Tile,
appProps: ApplicationProperties,
) {
val tileFilename = TilePngFilename(
tile.serverId,
tile.surfaceIndex,
tile.zoomLevel,
tile.chunkX,
tile.chunkY,
)
val tileFile = appProps.serverDataDir.combineSafe(Path(tileFilename.value))
if (tileFile.exists()) {
call.respondFile(tileFile) {
versions = versions + listOf(
EntityTagVersion(tileFile.lastModified().hashCode().toString()),
LastModifiedVersion(tileFile.lastModified()),
)
}
} else {
call.respond(HttpStatusCode.NotFound)
}
}
@Serializable
@Resource("kafkatorio/data/servers")
private class FactorioServers
@Serializable
@Resource("kafkatorio/data/servers/{factorioServerId}")
data class FactorioServer(
val factorioServerId: FactorioServerId,
) {
@Serializable
@Resource("map")
data class MapData(
val server: FactorioServer,
) {
@Serializable
@Resource("tiles/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png")
data class Tile(
val mapData: MapData,
val surfaceIndex: SurfaceIndex,
val zoomLevel: Int,
val chunkX: Int,
val chunkY: Int,
)
@Serializable
@Resource("buildings/s{surfaceIndex}")
data class Buildings(
val mapData: MapData,
val surfaceIndex: SurfaceIndex,
// a list of buildings... (Sealed interface, different types?)
)
}
@Serializable
@Resource("players")
data class Players(
val server: FactorioServer,
)
@Serializable
@Resource("{playerId}")
data class Player(
val server: FactorioServer,
val playerIndex: PlayerIndex,
)
}
private val FactorioServer.MapData.Tile.serverId: FactorioServerId
get() = mapData.server.factorioServerId

50
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/restApi.kt Normal file
View file

@ -0,0 +1,50 @@
package dev.adamko.kafkatorio.server.web.rest
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.serialization.kotlinx.json.json
import io.ktor.server.application.Application
import io.ktor.server.application.install
import io.ktor.server.application.log
import io.ktor.server.plugins.autohead.AutoHeadResponse
import io.ktor.server.plugins.callloging.CallLogging
import io.ktor.server.plugins.compression.Compression
import io.ktor.server.plugins.compression.excludeContentType
import io.ktor.server.plugins.compression.minimumSize
import io.ktor.server.plugins.conditionalheaders.ConditionalHeaders
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
import io.ktor.server.plugins.partialcontent.PartialContent
import io.ktor.server.plugins.statuspages.StatusPages
import io.ktor.server.resources.Resources
import io.ktor.server.response.respondText
import io.ktor.server.routing.routing
fun Application.webmapTileServer(appProps: ApplicationProperties) {
install(AutoHeadResponse)
install(CallLogging)
install(Compression) {
default()
minimumSize(1024)
excludeContentType(ContentType.Video.Any)
}
install(ConditionalHeaders)
install(ContentNegotiation) {
json()
}
install(PartialContent)
install(Resources)
install(StatusPages) {
exception<Throwable> { call, cause ->
this@webmapTileServer.log.error("$call caused exception - returning 500", cause)
call.respondText(text = "500: $cause", status = HttpStatusCode.InternalServerError)
}
}
routing {
factorioServersRoutes(appProps)
}
}

27
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/webServer.kt Normal file
View file

@ -0,0 +1,27 @@
package dev.adamko.kafkatorio.server.web
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.web.rest.webmapTileServer
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import dev.adamko.kafkatorio.server.web.websocket.webmapWebsocketServer
import io.ktor.server.cio.CIO
import io.ktor.server.engine.embeddedServer
import kotlinx.coroutines.coroutineScope
suspend fun startWebServer(
appProps: ApplicationProperties,
wsServer: WebmapWebsocketServer,
): Unit = coroutineScope {
embeddedServer(
factory = CIO,
port = appProps.webPort.value,
) {
webmapWebsocketServer(wsServer)
webmapTileServer(appProps)
}.start(wait = true)
}

112
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/WebmapWebsocketServer.kt Normal file
View file

@ -0,0 +1,112 @@
package dev.adamko.kafkatorio.server.web.websocket
import io.ktor.websocket.DefaultWebSocketSession
import io.ktor.websocket.Frame
import io.ktor.websocket.send
import java.util.concurrent.atomic.AtomicInteger
import kotlin.time.Duration.Companion.seconds
import kotlin.time.TimeSource
import kotlinx.coroutines.awaitCancellation
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.isActive
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
import kotlinx.coroutines.yield
class WebmapWebsocketServer {
private val startTime = TimeSource.Monotonic.markNow()
private val clients: MutableStateFlow<List<WSConnection>> = MutableStateFlow(emptyList())
private val messagesToClients: MutableSharedFlow<String> = MutableSharedFlow(
extraBufferCapacity = Int.MAX_VALUE,
onBufferOverflow = BufferOverflow.DROP_OLDEST
)
private val messagesFromClients: MutableSharedFlow<Frame> = MutableSharedFlow(
extraBufferCapacity = Int.MAX_VALUE,
onBufferOverflow = BufferOverflow.DROP_OLDEST
)
suspend fun bindClient(session: DefaultWebSocketSession): Unit = coroutineScope {
val client = WSConnection(session)
clients.update { clients -> clients + client }
client.coroutineContext.job.invokeOnCompletion { e ->
log("client ${client.name} completed ${e ?: "successfully"}")
clients.update { clients ->
(clients - client).filter { it.isActive }
}
}
messagesToClients.onEach { msg ->
runCatching {
if (client.isActive) {
client.outgoing.send(Frame.Text(msg))
}
}
}.launchIn(client)
client.launch {
client.incoming.receiveAsFlow().collect(messagesFromClients)
}
// client.launch {
while (isActive) {
runCatching {
client.send("elapsed ${startTime.elapsedNow()}")
}
delay(10.seconds)
yield()
}
// }
// awaitCancellation()
}
suspend fun sendMessageToClients(msg: String) {
log("sending message to clients: $msg")
messagesToClients.emit(msg)
}
private data class WSConnection(
private val session: DefaultWebSocketSession,
) : DefaultWebSocketSession by session {
val name: String = "user${lastId.getAndIncrement()}"
override suspend fun send(frame: Frame): Unit = runCatching {
super.send(frame)
}.fold(
onSuccess = { log("sent message to client $name: $frame") },
onFailure = { e -> log("failed to send message to client $name: $e") }
)
override fun equals(other: Any?): Boolean = name == other
override fun hashCode(): Int = name.hashCode()
companion object {
private val lastId = AtomicInteger(0)
}
}
companion object {
private fun log(msg: String) = println("[WebmapWebsocketServer] $msg")
}
}

34
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/wsServer.kt Normal file
View file

@ -0,0 +1,34 @@
package dev.adamko.kafkatorio.server.web.websocket
import io.ktor.server.application.Application
import io.ktor.server.application.install
import io.ktor.server.routing.routing
import io.ktor.server.websocket.WebSockets
import io.ktor.server.websocket.webSocket
import io.ktor.websocket.WebSocketDeflateExtension
import java.util.zip.Deflater
fun Application.webmapWebsocketServer(
wsServer: WebmapWebsocketServer,
) {
install(WebSockets) {
extensions {
install(WebSocketDeflateExtension) {
// Compression level to use for java.util.zip.Deflater
compressionLevel = Deflater.DEFAULT_COMPRESSION
// Prevent compressing small outgoing frames
compressIfBiggerThan(bytes = 4 * 1024)
}
}
}
routing {
// TODO only send a ws client message for a single Factorio-sever. Make the URL server specific.
// /kafkatorio/data/servers/{serverId}/ws
webSocket("kafkatorio/ws") {
wsServer.bindClient(this)
}
}
}

148
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/webmapTileServer.kt
View file

@ -1,148 +0,0 @@
package dev.adamko.kafkatorio.server
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.TilePngFilename
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.http.content.EntityTagVersion
import io.ktor.http.content.versions
import io.ktor.resources.Resource
import io.ktor.serialization.kotlinx.json.json
import io.ktor.server.application.Application
import io.ktor.server.application.ApplicationCall
import io.ktor.server.application.call
import io.ktor.server.application.install
import io.ktor.server.http.content.LastModifiedVersion
import io.ktor.server.plugins.autohead.AutoHeadResponse
import io.ktor.server.plugins.compression.Compression
import io.ktor.server.plugins.compression.excludeContentType
import io.ktor.server.plugins.compression.minimumSize
import io.ktor.server.plugins.conditionalheaders.ConditionalHeaders
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
import io.ktor.server.plugins.partialcontent.PartialContent
import io.ktor.server.resources.Resources
import io.ktor.server.resources.get
import io.ktor.server.response.respond
import io.ktor.server.response.respondFile
import io.ktor.server.routing.routing
import io.ktor.util.combineSafe
import io.ktor.util.pipeline.PipelineContext
import kotlin.io.path.Path
import kotlinx.serialization.Serializable
fun Application.webmapTileServer(appProps: ApplicationProperties) {
install(AutoHeadResponse)
install(Compression) {
default()
minimumSize(1024)
excludeContentType(ContentType.Video.Any)
}
install(ConditionalHeaders)
install(ContentNegotiation) {
json()
}
install(PartialContent)
install(Resources)
val server = WebmapTileServer(appProps)
routing {
with(server) {
get<FactorioServers.FactorioServer.MapData.Tile> { tile ->
handleTileRequest(tile)
}
get<FactorioServers> {
}
}
}
}
private class WebmapTileServer(
private val appProps: ApplicationProperties
) {
private val serverDataDir by appProps::serverDataDir
suspend fun PipelineContext<Unit, ApplicationCall>.handleTileRequest(
tile: FactorioServers.FactorioServer.MapData.Tile
) {
val tileFilename = TilePngFilename(
tile.serverId,
tile.surfaceIndex,
tile.zoomLevel,
tile.chunkX,
tile.chunkY,
)
val tileFile = serverDataDir.combineSafe(Path(tileFilename.value))
if (tileFile.exists()) {
call.respondFile(tileFile) {
versions = versions + listOf(
EntityTagVersion(tileFile.lastModified().hashCode().toString()),
LastModifiedVersion(tileFile.lastModified()),
)
}
} else {
call.respond(HttpStatusCode.NotFound)
}
}
}
@Serializable
@Resource("servers")
private class FactorioServers {
@Serializable
@Resource("{factorioServerId}")
data class FactorioServer(
val factorioServerId: FactorioServerId,
) {
@Serializable
@Resource("map")
class MapData {
@Serializable
@Resource("tiles/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png")
data class Tile(
val parent: FactorioServer,
val surfaceIndex: SurfaceIndex,
val zoomLevel: Int,
val chunkX: Int,
val chunkY: Int,
) {
val serverId: FactorioServerId by parent::factorioServerId
}
@Serializable
@Resource("buildings/s{surfaceIndex}")
data class Buildings(
val surfaceIndex: SurfaceIndex,
// a list of buildings... (Sealed interface, different types?)
)
}
@Serializable
@Resource("players")
class Players {
@Serializable
@Resource("{playerId}")
data class Id(
val playerIndex: PlayerIndex,
)
}
}
}

130
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/webmapWebsocketServer.kt
View file

@ -1,130 +0,0 @@
package dev.adamko.kafkatorio.server
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import io.ktor.server.application.Application
import io.ktor.server.application.install
import io.ktor.server.routing.routing
import io.ktor.server.websocket.WebSockets
import io.ktor.server.websocket.webSocket
import io.ktor.websocket.DefaultWebSocketSession
import io.ktor.websocket.Frame
import io.ktor.websocket.WebSocketDeflateExtension
import io.ktor.websocket.send
import java.util.concurrent.atomic.AtomicInteger
import java.util.zip.Deflater
import kotlin.time.Duration.Companion.seconds
import kotlin.time.TimeSource
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.isActive
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
fun Application.webmapWebsocketServer() {
install(WebSockets) {
extensions {
install(WebSocketDeflateExtension) {
// Compression level to use for java.util.zip.Deflater
compressionLevel = Deflater.DEFAULT_COMPRESSION
// Prevent compressing small outgoing frames
compressIfBiggerThan(bytes = 4 * 1024)
}
}
}
val wsServer = WebmapWebsocketServer()
routing {
webSocket("/ws") {
wsServer.bindClient(this)
}
}
}
private class WebmapWebsocketServer {
private val startTime = TimeSource.Monotonic.markNow()
private val clients: MutableStateFlow<Set<WSConnection>> = MutableStateFlow(emptySet())
private val messagesToClients: MutableSharedFlow<String> = MutableSharedFlow(
extraBufferCapacity = Int.MAX_VALUE,
onBufferOverflow = BufferOverflow.DROP_OLDEST
)
private val messagesFromClients: MutableSharedFlow<Frame> = MutableSharedFlow(
extraBufferCapacity = Int.MAX_VALUE,
onBufferOverflow = BufferOverflow.DROP_OLDEST
)
suspend fun bindClient(session: DefaultWebSocketSession) {
val client = WSConnection(session)
clients.update { clients -> clients + client }
messagesToClients.onEach { msg ->
runCatching {
client.send(msg)
}
}.launchIn(client)
client.launch {
client.incoming.receiveAsFlow()
.collect(messagesFromClients)
}
client.launch {
while (isActive) {
runCatching {
client.send("elapsed ${startTime.elapsedNow()}")
}
delay(10.seconds)
}
}
client.coroutineContext.job.invokeOnCompletion {
log("client ${client.name} completed")
clients.update { clients -> clients - client }
}
}
suspend fun sendMessageToClients(msg: String) {
log("sending message to clients: $msg")
messagesToClients.emit(msg)
}
}
private data class WSConnection(
private val session: DefaultWebSocketSession,
) : DefaultWebSocketSession by session {
val name: String = "user${lastId.getAndIncrement()}"
override suspend fun send(frame: Frame): Unit = runCatching {
super.send(frame)
}.fold(
onSuccess = { log("sent message to client $name: $frame") },
onFailure = { log("failed to send message to client $name: $frame") }
)
override fun equals(other: Any?): Boolean = name == other
override fun hashCode(): Int = name.hashCode()
companion object {
private val lastId = AtomicInteger(0)
}
}
private fun log(msg: String) = println("[WebmapWebsocketServer] $msg")

4
modules/events-server/src/main/resources/config.yml
View file

@ -1,11 +1,11 @@
# Factorio Events Processor config # Factorio Events Processor config
web-port: ${SERVER_PORT:-8080} web-port: ${SERVER_PORT:-8080}
servers-data-dir: ${SERVER_DATA_DIR:-./server-data} server-data-dir: ${SERVER_DATA_DIR:-./server-data}
socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1} socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}
socket-server-port: ${SOCKET_SERVER_PORT:-9002} socket-server-port: ${SOCKET_SERVER_PORT:-12092}
kafka-admin-config: kafka-admin-config:

49
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessorTest.kt
View file

@ -1,49 +0,0 @@
package dev.adamko.kafkatorio.processor.misc
import io.kotest.core.spec.style.FunSpec
import io.kotest.core.spec.style.Test
import io.mockk.every
import io.mockk.mockk
import kotlin.time.Duration.Companion.minutes
import org.apache.kafka.common.serialization.Serdes
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.state.KeyValueIterator
import org.apache.kafka.streams.state.TimestampedKeyValueStore
class DebounceProcessorTest : FunSpec({
}) {
@Test
fun blah() {
// val topology = Topology()
//
// topology.addProcessor(
// "blah",
// debounceProcessor(
// 1.minutes,
// "asdasda",
// Serdes.String(),
// Serdes.String()
// )
// )
// val streamsBuilder = StreamsBuilder()
//
// streamsBuilder
// .stream<String, String>("input-topic")
// .process(
// debounceProcessor(
// 1.minutes,
// "input-topic-debounce-store",
// Serdes.String(),
// Serdes.String(),
// ),
// )
}
}

6
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/serdeSize.kt
View file

@ -1,14 +1,14 @@
package dev.adamko.kafkatorio.processor.misc package dev.adamko.kafkatorio.processor.misc
import dev.adamko.kafkatorio.processor.serdes.kxsBinary import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
import dev.adamko.kafkatorio.processor.topology.ServerMapChunkTiles
import dev.adamko.kafkatorio.schema.common.ChunkSize import dev.adamko.kafkatorio.schema.common.ChunkSize
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.MapChunkPosition import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapTilePosition import dev.adamko.kafkatorio.schema.common.MapTilePosition
import dev.adamko.kafkatorio.schema.common.SurfaceIndex import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
import java.util.UUID import java.util.UUID
import kotlinx.serialization.encodeToByteArray import kotlinx.serialization.encodeToByteArray
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
@ -24,7 +24,7 @@ class Blah {
serverId = FactorioServerId(UUID.randomUUID().toString()), serverId = FactorioServerId(UUID.randomUUID().toString()),
chunkPosition = MapChunkPosition(Int.MAX_VALUE, Int.MAX_VALUE), chunkPosition = MapChunkPosition(Int.MAX_VALUE, Int.MAX_VALUE),
surfaceIndex = SurfaceIndex(UInt.MAX_VALUE), surfaceIndex = SurfaceIndex(UInt.MAX_VALUE),
chunkSize = ChunkSize.MAX chunkSize = ChunkSize.MAX,
), ),
buildMap { buildMap {
repeat(512) { x -> repeat(512) { x ->

6
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/topology/GroupMapChunksTest.kt
View file

@ -1,8 +1,8 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.processor.topology
//import dev.adamko.kafkatorio.processor.admin.TOPIC_GROUPED_MAP_CHUNKS_STATE //import dev.adamko.kafkatorio.server.processor.TOPIC_GROUPED_MAP_CHUNKS_STATE
//import dev.adamko.kafkatorio.processor.serdes.jsonMapper //import dev.adamko.kafkatorio.processor.config.getJsonMapper
//import dev.adamko.kafkatorio.processor.serdes.kxsBinary //import dev.adamko.kafkatorio.library.getKxsBinary
//import dev.adamko.kafkatorio.schema.common.Colour //import dev.adamko.kafkatorio.schema.common.Colour
//import dev.adamko.kafkatorio.schema.common.ColourHex //import dev.adamko.kafkatorio.schema.common.ColourHex
//import dev.adamko.kafkatorio.schema.common.MapChunkPosition //import dev.adamko.kafkatorio.schema.common.MapChunkPosition

4
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/topology/SaveMapChunksTest.kt → modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/server/processor/topology/SaveMapChunksTest.kt
View file

@ -1,6 +1,6 @@
package dev.adamko.kafkatorio.processor.topology package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.processor.serdes.kxsBinary import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.schema.common.ChunkSize import dev.adamko.kafkatorio.schema.common.ChunkSize
import dev.adamko.kafkatorio.schema.common.ColourHex import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioServerId import dev.adamko.kafkatorio.schema.common.FactorioServerId

2
modules/infra-factorio-server/src/docker-compose.yml
View file

@ -19,7 +19,7 @@ services:
logging: logging:
driver: syslog driver: syslog
options: options:
syslog-address: "tcp://host.docker.internal:9002" syslog-address: "tcp://host.docker.internal:12092"
syslog-format: "rfc5424" syslog-format: "rfc5424"
# environment: # environment:

50
modules/web-map/build.gradle.kts
View file

@ -17,23 +17,6 @@ kvision {
} }
rootProject.extensions.configure<NodeJsRootExtension> {
// https://github.com/rjaros/kvision/issues/410
versions.webpackCli.version = "4.10.0"
versions.webpackDevServer.version = "4.9.2"
}
rootProject.extensions.configure<YarnRootExtension> {
resolution("http-proxy-middleware", "^2.0.6")
}
//rootProject.afterEvaluate {
// extensions.configure<NodeJsRootExtension> {
// // https://github.com/rjaros/kvision/issues/410
// versions.webpackCli.version = "4.10.0"
// }
//}
afterEvaluate { afterEvaluate {
rootProject.extensions.configure<NodeJsRootExtension> { rootProject.extensions.configure<NodeJsRootExtension> {
// https://github.com/rjaros/kvision/issues/410 // https://github.com/rjaros/kvision/issues/410
@ -58,12 +41,12 @@ kotlin {
open = false, open = false,
port = 3000, port = 3000,
proxy = mutableMapOf( proxy = mutableMapOf(
"/tiles" to mapOf( "/kafkatorio/data/servers" to mapOf(
"target" to "http://localhost:12080", "target" to "http://localhost:12080",
"secure" to false, "secure" to false,
"changeOrigin" to true, "changeOrigin" to true,
), ),
"/ws/foo" to mapOf( "/kafkatorio/ws" to mapOf(
"target" to "ws://localhost:12080", "target" to "ws://localhost:12080",
// "secure" to false, // "secure" to false,
"ws" to true, "ws" to true,
@ -104,12 +87,10 @@ kotlin {
implementation(libs.kotlinx.coroutines.core) implementation(libs.kotlinx.coroutines.core)
implementation(projects.modules.eventsSchema) implementation(projects.modules.eventsLibrary)
implementation(libs.kotlinx.html) implementation(libs.kotlinx.html)
implementation(projects.modules.eventsSchema)
implementation(libs.kotlinx.nodejs) implementation(libs.kotlinx.nodejs)
implementation(libs.kotlinx.html) implementation(libs.kotlinx.html)
@ -176,28 +157,3 @@ fun KotlinDependencyHandler.kvision(
// resolution("webpack-dev-server", "4.9.1") // resolution("webpack-dev-server", "4.9.1")
// } // }
//} //}
//evaluationDependsOn(rootProject.path)
//evaluationDependsOn(projects.modules.eventsSchema.dependencyProject.path)
//// stop warnings when building
//tasks.configureEach {
// when (name) {
// "productionExecutableCompileSync" ->
// dependsOn(taskProvider("compileProductionExecutableKotlinJs"))
// "testTestDevelopmentExecutableCompileSync" ->
// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinJs"))
//// "jsTestTestDevelopmentExecutableCompileSync" ->
//// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinFrontend"))
//// "jsProductionExecutableCompileSync" ->
//// dependsOn(taskProvider("compileProductionExecutableKotlinFrontend"))
// "browserDevelopmentRun" ->
// dependsOn(taskProvider("browserDevelopmentWebpack"))
// "browserDevelopmentWebpack" ->
// dependsOn(taskProvider("assemble"))
// "browserProductionWebpack" ->
// dependsOn(taskProvider("developmentExecutableCompileSync"))
// }
//}

9
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/App.kt
View file

@ -77,12 +77,9 @@ class App(
wsService.packetsFlow wsService.packetsFlow
.filterIsInstance<EventServerPacket.ChunkTileSaved>() .filterIsInstance<EventServerPacket.ChunkTileSaved>()
.onEach { .onEach { tileSavedEvent ->
println("[packetsFlow] triggering tile refresh ${it.filename.value}") println("[packetsFlow] triggering tile refresh ${tileSavedEvent.filename.value}")
gameState.map.refreshUpdatedTilePng( gameState.map.refreshUpdatedTilePng(tileSavedEvent.filename)
// doc, window,
it.filename
)
}.launchIn(this) }.launchIn(this)
} }

2
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/ApplicationProperties.kt
View file

@ -8,5 +8,5 @@ object Props {
get() = window.location.href get() = window.location.href
.replace("http://", "ws://") .replace("http://", "ws://")
.replace("https://", "wss://") + .replace("https://", "wss://") +
"ws/foo" "kafkatorio/ws"
} }

1
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/externals/leaflet.kt vendored
View file

@ -14,6 +14,7 @@ inline var TileLayer<TileLayer.TileLayerOptions>.tileOnLoad: TileOnLoadFn?
} }
/** @see [tileOnLoad] */
typealias TileOnLoadFn = (done: DoneCallback, tile: HTMLElement) -> Unit typealias TileOnLoadFn = (done: DoneCallback, tile: HTMLElement) -> Unit

2
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/json.kt
View file

@ -1,6 +1,6 @@
package dev.adamko.kafkatorio.webmap package dev.adamko.kafkatorio.webmap
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
val jsonMapper = Json { val jsonMapper = Json {

6
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt
View file

@ -31,7 +31,7 @@ import org.w3c.fetch.RequestMode
class FactorioMap( class FactorioMap(
private val tileUrlTemplate: String = """/tiles/s1/z{z}/x{x}/y{y}.png""", private val tileUrlTemplate: String = """/kafkatorio/data/servers/syslog-test/map/tiles/s1/z{z}/x{x}/y{y}.png""",
) { ) {
val kvMap: Maps = Maps { val kvMap: Maps = Maps {
@ -89,9 +89,7 @@ class FactorioMap(
.querySelectorAll("img.leaflet-tile-loaded") .querySelectorAll("img.leaflet-tile-loaded")
.asList() .asList()
.filterIsInstance<Image>() .filterIsInstance<Image>()
.filter { img -> .filter { img -> tilePngFilename.value in img.src }
tilePngFilename.value in img.src
}
.forEach { img -> .forEach { img ->
val imgSrc = img.src val imgSrc = img.src

3
settings.gradle.kts
View file

@ -2,8 +2,7 @@ rootProject.name = "kafkatorio"
include( include(
":modules:events-mod", ":modules:events-mod",
":modules:events-schema", ":modules:events-library",
":modules:events-processor",
":modules:events-server", ":modules:events-server",
":modules:infra-factorio-client", ":modules:infra-factorio-client",