lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

big ol' refactor to split up the events-server and events-processors

Browse source
This commit is contained in:
Adam 2022-09-13 17:56:57 +02:00
parent d82f25c6ee
commit 873983f9a3
71 changed files with 1013 additions and 757 deletions
Download patch file Download diff file Expand all files Collapse all files

91
modules/events-processors/build.gradle.kts Normal file
View file

@ -0,0 +1,91 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
description = "processors for Kafkatorio packets"
val projectId: String by project.extra
dependencies {
implementation(platform(projects.modules.versionsPlatform))
implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(projects.modules.eventsProcessorCore)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.logging)
implementation(libs.scrimage.core)
implementation(libs.okio.core)
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams)
}
application {
mainClass.set("dev.adamko.kafkatorio.processors.MainKt")
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
"-opt-in=kotlinx.coroutines.FlowPreview",
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}
//val startScriptEntitiesProcessor = registerStartScriptTask("EntitiesProcessor") {
// mainClass.set("dev.adamko.kafkatorio.processors.EntitiesProcessorKt")
//}
//
//val startScriptPacketsProcessor = registerStartScriptTask("PacketsProcessor") {
// mainClass.set("dev.adamko.kafkatorio.processors.PacketsProcessorKt")
//}
//
//val startScriptTilesProcessor = registerStartScriptTask("TilesProcessor") {
// mainClass.set("dev.adamko.kafkatorio.processors.TilesProcessorKt")
//}
//
//
//application.applicationDistribution.into("bin") {
// from(startScriptEntitiesProcessor)
// from(startScriptPacketsProcessor)
// from(startScriptTilesProcessor)
// fileMode = 0b000_111_101_101 // 0755
// duplicatesStrategy = DuplicatesStrategy.EXCLUDE
//}
tasks.named("run") {
val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka/")
doFirst {
kafkaStateDir.asFile.mkdirs()
}
}
val runEventsProcessors by tasks.registering {
group = rootProject.name
dependsOn(tasks.run)
}

54
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/groupEntitiesMapChunks.kt Normal file
View file

@ -0,0 +1,54 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.core.launchTopology
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.colourEntityChunks
import dev.adamko.kafkatorio.server.processor.topology.convertEntityUpdateToServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.convertResourceUpdateToServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.KStream
/** Building and resource tiles */
suspend fun groupEntitiesMapChunks(
layer: ServerMapTileLayer,
colouredUpdatesStreamTopic: String,
colouredChunkStateTopic: String,
) {
val builder = StreamsBuilder()
val entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> =
when (layer) {
ServerMapTileLayer.TERRAIN -> return
ServerMapTileLayer.RESOURCE ->
builder.streamPacketData<MapChunkResourceUpdate>()
.convertResourceUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
ServerMapTileLayer.BUILDING ->
builder.streamPacketData<MapChunkEntityUpdate>()
.convertEntityUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
}
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
builder.streamPacketData()
val topology = colourEntityChunks(
builder,
entityChunksStream,
protosStream,
layer,
colouredUpdatesStreamTopic,
colouredChunkStateTopic,
)
launchTopology("groupTilesMapChunks.${layer.dir}", topology)
}

23
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/groupTilesMapChunks.kt Normal file
View file

@ -0,0 +1,23 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.core.launchTopology
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.KStream
/** Terrain tiles */
suspend fun groupTilesMapChunks() {
val builder = StreamsBuilder()
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
builder.streamPacketData()
val topology = colourMapChunks(builder, protosStream)
launchTopology("groupTilesMapChunks.${ServerMapTileLayer.TERRAIN.dir}", topology)
}

36
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/main.kt Normal file
View file

@ -0,0 +1,36 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
suspend fun main(): Unit = coroutineScope {
launch {
splitPackets()
}
launch {
groupTilesMapChunks()
}
launch {
groupEntitiesMapChunks(
layer = ServerMapTileLayer.BUILDING,
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES,
colouredChunkStateTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
)
}
launch {
groupEntitiesMapChunks(
layer = ServerMapTileLayer.RESOURCE,
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES,
colouredChunkStateTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
)
}
}

17
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/splitPackets.kt Normal file
View file

@ -0,0 +1,17 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.core.launchTopology
import dev.adamko.kafkatorio.server.processor.topology.factorioServerPacketStream
import dev.adamko.kafkatorio.server.processor.topology.splitFactorioServerPacketStream
import org.apache.kafka.streams.StreamsBuilder
suspend fun splitPackets() {
val builder = StreamsBuilder()
val packets = factorioServerPacketStream(builder)
splitFactorioServerPacketStream(packets)
val topology = builder.build()
launchTopology("splitPackets", topology)
}