lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

big ol' refactor to split up the events-server and events-processors

Browse source
This commit is contained in:
Adam 2022-09-13 17:56:57 +02:00
parent d82f25c6ee
commit 873983f9a3
71 changed files with 1013 additions and 757 deletions
Download patch file Download diff file Expand all files Collapse all files

112
.gitignore vendored
View file

@ -2,10 +2,12 @@
# ignore any files that start with '.secret'
.secret.*
.idea
.state/
.idea/
!.idea/icon.svg
node_modules
node_modules/
*.hprof
@ -16,108 +18,13 @@ node_modules
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# AWS User-specific
.idea/**/aws.xml
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
.idea/artifacts
.idea/compiler.xml
.idea/jarRepositories.xml
.idea/modules.xml
.idea/*.iml
.idea/modules
# File-based project format
*.iws
*.iml
*.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
### Intellij Patch ###
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
# *.iml
# modules.xml
# .idea/misc.xml
# *.ipr
# Sonarlint plugin
# https://plugins.jetbrains.com/plugin/7973-sonarlint
.idea/**/sonarlint/
# SonarQube Plugin
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
.idea/**/sonarIssues.xml
# Markdown Navigator plugin
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
.idea/**/markdown-navigator.xml
.idea/**/markdown-navigator-enh.xml
.idea/**/markdown-navigator/
# Cache file creation bug
# See https://youtrack.jetbrains.com/issue/JBR-2257
.idea/$CACHE_FILE$
# CodeStream plugin
# https://plugins.jetbrains.com/plugin/12206-codestream
.idea/codestream.xml
### Kotlin ###
# Compiled class file
*.class
@ -125,12 +32,6 @@ fabric.properties
# Log file
*.log
# BlueJ files
*.ctxt
# Mobile Tools for Java (J2ME)
.mtj.tmp/
# Package Files #
*.jar
*.war
@ -198,7 +99,6 @@ Temporary Items
# Local History for Visual Studio Code
.history/
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
.ionide

70
build.gradle.kts
View file

@ -41,17 +41,17 @@ tasks.wrapper {
}
val startInfra by tasks.registering {
group = project.name
group = rootProject.name
dependsOn(
// ":modules:infra-kafka-pipe:processRun",
runKafkaCluster
// runKafkaCluster
// ":modules:infra-kafka-connect:processRun",
)
}
val runKafkatorio by tasks.registering {
group = project.name
group = rootProject.name
dependsOn(
":modules:infra-factorio-client:processRestart",
@ -60,44 +60,44 @@ val runKafkatorio by tasks.registering {
)
}
val runEventsServer by tasks.registering {
group = project.name
//val runEventsServer by tasks.registering {
// group = project.name
//
// dependsOn(":modules:events-server:run")
//}
dependsOn(":modules:events-server:run")
}
//val runWebMap by tasks.registering {
// group = project.name
//
// dependsOn(":modules:web-map:browserDevelopmentRun")
//}
val runWebMap by tasks.registering {
group = project.name
dependsOn(":modules:web-map:browserDevelopmentRun")
}
//
//val runKafkaCluster by tasks.registering {
// group = project.name
//
// dependsOn(
// ":modules:infra-kafka-cluster:processRun",
// )
//}
val runKafkaCluster by tasks.registering {
group = project.name
dependsOn(
":modules:infra-kafka-cluster:processRun",
)
}
//val runFactorioServer by tasks.registering {
// group = project.name
//
// dependsOn(
// ":modules:infra-factorio-server:processRun",
// )
//}
val runFactorioServer by tasks.registering {
group = project.name
dependsOn(
":modules:infra-factorio-server:processRun",
)
}
val runFactorioClient by tasks.registering {
group = project.name
dependsOn(
":modules:infra-factorio-client:processRun",
)
}
//val runFactorioClient by tasks.registering {
// group = project.name
//
// dependsOn(
// ":modules:infra-factorio-client:processRun",
// )
//}
apply(from = "$projectDir/kt52647.gradle.kts")

2
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/base.gradle.kts
View file

@ -1,7 +1,5 @@
package dev.adamko.kafkatorio
import org.gradle.api.internal.tasks.userinput.UserInputHandler
import org.gradle.kotlin.dsl.support.serviceOf
plugins {
base

22
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/gradle/extensions.kt
View file

@ -7,9 +7,13 @@ import org.gradle.api.file.Directory
import org.gradle.api.file.FileCopyDetails
import org.gradle.api.file.ProjectLayout
import org.gradle.api.file.RelativePath
import org.gradle.api.plugins.JavaPlugin
import org.gradle.api.provider.Provider
import org.gradle.api.specs.NotSpec
import org.gradle.api.specs.Spec
import org.gradle.api.tasks.TaskProvider
import org.gradle.api.tasks.application.CreateStartScripts
import org.gradle.kotlin.dsl.register
import org.gradle.plugins.ide.idea.model.IdeaModule
import org.gradle.process.ExecSpec
import org.jetbrains.kotlin.gradle.targets.js.yarn.YarnRootExtension
@ -94,3 +98,21 @@ fun Project.taskProvider(taskName: String): Provider<Task> = providers.provider
const val DOCKER_COMPOSE_TASK_GROUP = "docker-compose"
fun Project.registerStartScriptTask(
appName: String,
configureStartScripts: CreateStartScripts.() -> Unit = {},
): TaskProvider<CreateStartScripts> =
tasks.register<CreateStartScripts>("startScript$appName") {
val jarTask = tasks.getByName(JavaPlugin.JAR_TASK_NAME)
val runtimeClasspath = configurations.getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME)
dependsOn(jarTask)
dependsOn(runtimeClasspath)
applicationName = appName
outputDir = layout.buildDirectory.dir("scripts").get().asFile
classpath = jarTask.outputs.files + runtimeClasspath
configureStartScripts()
}

2
gradle/libs.versions.toml
View file

@ -11,7 +11,7 @@ kotlin = "1.7.10"
## libs ##
camel-kafkaConnectors = "0.11.0"
http4k = "4.25.16.2"
kafka = "3.2.1" # https://kafka.apache.org/downloads
kafka = "3.2.2" # https://kafka.apache.org/downloads
kvision = "5.14.0" # https://github.com/rjaros/kvision/releases
log4j2 = "2.17.2"
okio = "3.2.0" # https://search.maven.org/artifact/com.squareup.okio/okio

24
modules/events-processor-core/build.gradle.kts
View file

@ -4,15 +4,15 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
// application
}
//description = "serves Kafkatorio content over the web"
description = "Shared Kafkatorio processors functionality"
val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka")
dependencies {
@ -37,12 +37,17 @@ dependencies {
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams)
implementation(libs.bundles.hoplite)
implementation(libs.ktorServer.authJwt)
// implementation(libs.nimbusJoseJwt)
}
application {
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
}
//application {
// mainClass.set("dev.adamko.kafkatorio.processor.admin.MainKt")
//}
tasks.withType<KotlinCompile>().configureEach {
@ -52,3 +57,10 @@ tasks.withType<KotlinCompile>().configureEach {
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}
val createKafkaTopics by tasks.registering(JavaExec::class) {
group = rootProject.name
classpath = sourceSets.main.get().runtimeClasspath
mainClass.set("dev.adamko.kafkatorio.processor.admin.MainKt")
}

8
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/admin/KafkatorioKafkaAdmin.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/KafkatorioKafkaAdmin.kt
View file

@ -1,7 +1,7 @@
package dev.adamko.kafkatorio.server.processor.admin
package dev.adamko.kafkatorio.processor.admin
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.processor.allTopics
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.config.allTopics
import java.util.concurrent.TimeUnit
import mu.KotlinLogging
import org.apache.kafka.clients.admin.Admin
@ -11,7 +11,7 @@ import org.apache.kafka.clients.admin.TopicListing
class KafkatorioKafkaAdmin(
appProps: ApplicationProperties
appProps: ApplicationProperties,
) {
private val logger = KotlinLogging.logger {}

10
modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/main.kt Normal file
View file

@ -0,0 +1,10 @@
package dev.adamko.kafkatorio.processor.admin
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
fun main() {
val appProps = ApplicationProperties.load()
val admin = KafkatorioKafkaAdmin(appProps)
admin.createKafkatorioTopics()
}

10
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/config/ApplicationProperties.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt
View file

@ -1,15 +1,16 @@
package dev.adamko.kafkatorio.server.config
package dev.adamko.kafkatorio.processor.config
import com.sksamuel.hoplite.ConfigLoaderBuilder
import com.sksamuel.hoplite.addEnvironmentSource
import com.sksamuel.hoplite.addResourceOrFileSource
import com.sksamuel.hoplite.sources.UserSettingsPropertySource
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.FactorioServerToken
import dev.adamko.kafkatorio.schema.common.MaskedValue
import java.nio.file.Path
val appProps = ApplicationProperties.load()
// loaded by hoplite
data class ApplicationProperties(
val webPort: PortVal,
val serverDataDir: Path,
@ -32,8 +33,13 @@ data class ApplicationProperties(
.allowUnresolvedSubstitutions()
.allowEmptySources()
.addEnvironmentSource()
.addResourceOrFileSource(
resourceOrFile = "${System.getProperty("user.home")}/.secret.kafkatorio-config.yml",
optional = true
)
.addResourceOrFileSource("/.secret.config.yml", optional = true)
.addResourceOrFileSource("/config.yml", optional = false)
.addPropertySource(UserSettingsPropertySource)
.build()
.loadConfigOrThrow()
}

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/config/PortVal.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/config/PortVal.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.config
package dev.adamko.kafkatorio.processor.config
/** Improved (slightly) version of http4k's `Port`. */

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/kafkatorioTopics.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/config/kafkatorioTopics.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.processor
package dev.adamko.kafkatorio.processor.config
import dev.adamko.kafkatorio.schema.packets.ConfigurationUpdate
import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate

12
modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/config/kxsJson.kt Normal file
View file

@ -0,0 +1,12 @@
package dev.adamko.kafkatorio.processor.config
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
import kotlinx.serialization.json.Json
@PublishedApi
internal val jsonMapper: Json = Json {
prettyPrint = true
prettyPrintIndent = " "
serializersModule = jsonMapperKafkatorio.serializersModule
}

4
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/security/Authenticator.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/core/Authenticator.kt
View file

@ -1,10 +1,10 @@
package dev.adamko.kafkatorio.server.security
package dev.adamko.kafkatorio.processor.core
import com.auth0.jwt.JWT
import com.auth0.jwt.algorithms.Algorithm
import com.auth0.jwt.interfaces.DecodedJWT
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.schema.common.FactorioServerToken
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import kotlin.reflect.jvm.jvmName
import kotlinx.coroutines.CancellationException
import kotlinx.serialization.SerialName

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/DebounceProcessor.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/core/DebounceProcessor.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.processor
package dev.adamko.kafkatorio.processor.core
import dev.adamko.kotka.extensions.component1
import dev.adamko.kotka.extensions.component2

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/StreamsExceptionHandler.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/core/StreamsExceptionHandler.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.processor
package dev.adamko.kafkatorio.processor.core
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse

88
modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/processor/core/topologyLauncher.kt Normal file
View file

@ -0,0 +1,88 @@
package dev.adamko.kafkatorio.processor.core
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import java.util.Properties
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
import kotlin.io.path.Path
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.cancel
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import org.apache.kafka.streams.KafkaStreams
import org.apache.kafka.streams.StreamsConfig
import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.TopologyDescription
private val appProps = ApplicationProperties.load()
suspend fun launchTopology(
id: String,
topology: Topology,
additionalProperties: Map<String, String> = mapOf(),
): Unit = coroutineScope {
val props: Properties = appProps.kafkaStreamsConfig.toProperties()
additionalProperties.forEach { (k, v) -> props.setProperty(k, v) }
val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
launch(CoroutineName(appId)) {
val streamsState: KafkaStreams.State = suspendCancellableCoroutine { cont ->
val streams = KafkaStreams(topology, props)
streams.setUncaughtExceptionHandler(StreamsExceptionHandler { e ->
cont.resumeWithException(e)
})
streams.setStateListener { newState: KafkaStreams.State, _ ->
when (newState) {
KafkaStreams.State.PENDING_SHUTDOWN,
KafkaStreams.State.NOT_RUNNING,
KafkaStreams.State.PENDING_ERROR,
KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
else -> Unit // do nothing
}
}
cont.invokeOnCancellation {
println("[$appId] coroutine cancelled -> closing. cause:$it")
streams.close()
}
printTopologyDescription(appId, topology)
streams.start()
println("launched Topology $appId")
}
println("exiting Topology $appId: $streamsState")
currentCoroutineContext().job.cancel("$appId: $streamsState")
}
}
private fun printTopologyDescription(appId: String, topology: Topology) {
val description: TopologyDescription = topology.describe()
val descFile = Path("./build/$appId.txt").toFile().apply {
parentFile.mkdirs()
if (!exists()) createNewFile()
writeText(description.toString())
}
println(
"""
|----------------
|$appId - ${descFile.canonicalPath}
|$description
|----------------
""".trimMargin()
)
}

6
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/colourMapChunks.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/colourMapChunks.kt
View file

@ -3,6 +3,7 @@ package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.library.resize
import dev.adamko.kafkatorio.library.toMapChunkPosition
import dev.adamko.kafkatorio.processor.core.DebounceProcessor.Companion.addDebounceProcessor
import dev.adamko.kafkatorio.schema.common.ChunkSize
import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioServerId
@ -12,9 +13,8 @@ import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.server.processor.DebounceProcessor.Companion.addDebounceProcessor
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.groupedAs
import dev.adamko.kotka.extensions.materializedAs

6
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/colourResourceChunks.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/colourResourceChunks.kt
View file

@ -2,6 +2,7 @@ package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.library.toMapTilePosition
import dev.adamko.kafkatorio.processor.core.DebounceProcessor.Companion.addDebounceProcessor
import dev.adamko.kafkatorio.schema.common.ChunkSize
import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.FactorioPrototype
@ -16,7 +17,6 @@ import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.server.processor.DebounceProcessor.Companion.addDebounceProcessor
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.materializedAs
import dev.adamko.kotka.extensions.stream
@ -27,7 +27,6 @@ import dev.adamko.kotka.extensions.tables.join
import dev.adamko.kotka.kxs.serde
import kotlin.collections.component1
import kotlin.collections.component2
import kotlin.math.ceil
import kotlin.math.floor
import kotlin.time.Duration.Companion.seconds
import org.apache.kafka.streams.StreamsBuilder
@ -232,6 +231,9 @@ private fun KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.enr
private fun MapBoundingBox.iterator(origin: MapTilePosition): Iterator<MapTilePosition> {
// check if tile width/height should be divided by 2. I think I read that it's not divided by 2
// if it's an odd number?
val xOffset: Int = floor(tileWidth.toDouble() / 2).toInt()
val startX = origin.x - xOffset
val endX = origin.x + xOffset

6
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/factorioServerPackets.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/factorioServerPackets.kt
View file

@ -4,9 +4,9 @@ import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.server.processor.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.server.processor.topicName
import dev.adamko.kafkatorio.processor.config.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.processor.config.jsonMapper
import dev.adamko.kafkatorio.processor.config.topicName
import dev.adamko.kotka.extensions.component1
import dev.adamko.kotka.extensions.component2
import dev.adamko.kotka.extensions.consumedAs

0
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/model.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/model.kt
View file

4
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/packetDataStreams.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/packetDataStreams.kt
View file

@ -1,10 +1,10 @@
package dev.adamko.kafkatorio.server.processor.topology
import dev.adamko.kafkatorio.processor.config.jsonMapper
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.server.processor.topicName
import dev.adamko.kafkatorio.processor.config.topicName
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.streams.filter
import dev.adamko.kotka.extensions.streams.mapValues

0
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/prototypes.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/prototypes.kt
View file

14
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/saveMapTiles.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/saveMapTiles.kt
View file

@ -10,11 +10,11 @@ import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
import dev.adamko.kafkatorio.schema.packets.EventServerPacket
import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.server.processor.TOPIC_BROADCAST_TO_WEBSOCKET
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
import dev.adamko.kafkatorio.processor.config.TOPIC_BROADCAST_TO_WEBSOCKET
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
import dev.adamko.kafkatorio.processor.config.jsonMapper
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.stream
@ -44,7 +44,7 @@ private const val WEB_MAP_TILE_SIZE_PX = 256
fun saveMapTiles(
builder: StreamsBuilder,
serverDataDir: Path,
): Topology {
) {
val subdividedMapChunkTilesDebounced: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
builder.stream(
@ -64,8 +64,6 @@ fun saveMapTiles(
subdividedMapChunkTilesDebounced.saveChunkAsImage(serverDataDir)
savedChunkFilenames.broadcastSavedTileUpdates()
return builder.build()
}

14
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/websocketBroadcaster.kt → modules/events-processor-core/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/websocketBroadcaster.kt
View file

@ -6,9 +6,8 @@ import dev.adamko.kafkatorio.schema.packets.EventServerPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
import dev.adamko.kafkatorio.schema.packets.SurfaceUpdate
import dev.adamko.kafkatorio.server.processor.TOPIC_BROADCAST_TO_WEBSOCKET
import dev.adamko.kafkatorio.server.processor.topicName
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import dev.adamko.kafkatorio.processor.config.TOPIC_BROADCAST_TO_WEBSOCKET
import dev.adamko.kafkatorio.processor.config.topicName
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.stream
@ -31,8 +30,13 @@ private val jsonMapper: Json = Json {
private const val pid = "websocketBroadcast"
interface WebsocketBroadcaster {
suspend fun sendMessageToClients(message: String)
}
fun broadcastToWebsocket(
websocketServer: WebmapWebsocketServer,
websocketBroadcaster: WebsocketBroadcaster,
builder: StreamsBuilder,
) {
@ -45,7 +49,7 @@ fun broadcastToWebsocket(
"$pid.websocket-server.send-message"
) { _, esPacket ->
runBlocking {
websocketServer.sendMessageToClients(jsonMapper.encodeToString(esPacket))
websocketBroadcaster.sendMessageToClients(jsonMapper.encodeToString(esPacket))
}
}
}

6
modules/events-server/src/main/resources/config.yml → modules/events-processor-core/src/main/resources/config.yml
View file

@ -5,7 +5,7 @@ server-data-dir: ${SERVER_DATA_DIR:-./server-data}
socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}
#socket-server-port: ${SOCKET_SERVER_PORT:-12092}
socket-server-port: ${SOCKET_SERVER_PORT:-12092}
kafka-admin-config:
@ -16,7 +16,7 @@ kafka-admin-config:
kafka-streams-config:
application.id: kafkatorio-events-processor
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}
state.dir: ${KAFKA_STREAMS_STATE_DIR:-./kafka-state}
state.dir: ${KAFKA_STREAMS_STATE_DIR:-.state/kafka}
num.stream.threads: 1
processing.guarantee: exactly_once_v2
auto.offset.reset: earliest
@ -65,5 +65,3 @@ kafka-streams-config:
# # fetch.message.max.bytes=31457280
# # 100mb 104857600
#
#tileServer:
# webmapTileDir: 'D:\Users\Adam\Projects\games\kafkatorio\modules\events-server\kafkatorio-tiles'

0
modules/events-server/src/main/resources/external-annotations/org/apache/kafka/streams/state/annotations.xml → modules/events-processor-core/src/main/resources/external-annotations/org/apache/kafka/streams/state/annotations.xml
View file

0
modules/events-server/src/main/resources/logback.xml → modules/events-processor-core/src/main/resources/logback.xml
View file

2
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/server/security/AuthenticatorTest.kt → modules/events-processor-core/src/test/kotlin/dev/adamko/kafkatorio/processor/core/AuthenticatorTest.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.security
package dev.adamko.kafkatorio.processor.core
import dev.adamko.kafkatorio.schema.common.MaskedValue
import dev.adamko.kafkatorio.server.config.ApplicationProperties

0
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/serdeSize.kt → modules/events-processor-core/src/test/kotlin/dev/adamko/kafkatorio/processor/core/misc/serdeSize.kt
View file

2
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/server/processor/topology/FactorioServerPacketsTest.kt → modules/events-processor-core/src/test/kotlin/dev/adamko/kafkatorio/processor/core/topology/FactorioServerPacketsTest.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.processor.topology
package dev.adamko.kafkatorio.processor.core.topology
import dev.adamko.kafkatorio.library.kxsBinary
import dev.adamko.kafkatorio.schema.common.ColourHex

2
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/topology/GroupMapChunksTest.kt → modules/events-processor-core/src/test/kotlin/dev/adamko/kafkatorio/processor/core/topology/GroupMapChunksTest.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.processor.topology
package dev.adamko.kafkatorio.processor.core.topology
//import dev.adamko.kafkatorio.server.processor.TOPIC_GROUPED_MAP_CHUNKS_STATE
//import dev.adamko.kafkatorio.processor.config.getJsonMapper

2
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/server/processor/topology/SaveMapChunksTest.kt → modules/events-processor-core/src/test/kotlin/dev/adamko/kafkatorio/processor/core/topology/SaveMapChunksTest.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.processor.topology
package dev.adamko.kafkatorio.processor.core.topology
import dev.adamko.kafkatorio.library.iterateTiles
import dev.adamko.kafkatorio.library.kxsBinary

54
modules/events-processor-entities/build.gradle.kts
View file

@ -1,54 +0,0 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
//description = "serves Kafkatorio content over the web"
val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
dependencies {
implementation(platform(projects.modules.versionsPlatform))
implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.logging)
implementation(libs.scrimage.core)
implementation(libs.okio.core)
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams)
}
application {
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
"-opt-in=kotlinx.coroutines.FlowPreview",
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}

54
modules/events-processor-packets/build.gradle.kts
View file

@ -1,54 +0,0 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
//description = "serves Kafkatorio content over the web"
val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
dependencies {
implementation(platform(projects.modules.versionsPlatform))
implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.logging)
implementation(libs.scrimage.core)
implementation(libs.okio.core)
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams)
}
application {
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
"-opt-in=kotlinx.coroutines.FlowPreview",
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}

54
modules/events-processor-tiles/build.gradle.kts
View file

@ -1,54 +0,0 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
//description = "serves Kafkatorio content over the web"
val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
dependencies {
implementation(platform(projects.modules.versionsPlatform))
implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.logging)
implementation(libs.scrimage.core)
implementation(libs.okio.core)
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams)
}
application {
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
"-opt-in=kotlinx.coroutines.FlowPreview",
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}

91
modules/events-processors/build.gradle.kts Normal file
View file

@ -0,0 +1,91 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
description = "processors for Kafkatorio packets"
val projectId: String by project.extra
dependencies {
implementation(platform(projects.modules.versionsPlatform))
implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(projects.modules.eventsProcessorCore)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.logging)
implementation(libs.scrimage.core)
implementation(libs.okio.core)
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams)
}
application {
mainClass.set("dev.adamko.kafkatorio.processors.MainKt")
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
"-opt-in=kotlinx.coroutines.FlowPreview",
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}
//val startScriptEntitiesProcessor = registerStartScriptTask("EntitiesProcessor") {
// mainClass.set("dev.adamko.kafkatorio.processors.EntitiesProcessorKt")
//}
//
//val startScriptPacketsProcessor = registerStartScriptTask("PacketsProcessor") {
// mainClass.set("dev.adamko.kafkatorio.processors.PacketsProcessorKt")
//}
//
//val startScriptTilesProcessor = registerStartScriptTask("TilesProcessor") {
// mainClass.set("dev.adamko.kafkatorio.processors.TilesProcessorKt")
//}
//
//
//application.applicationDistribution.into("bin") {
// from(startScriptEntitiesProcessor)
// from(startScriptPacketsProcessor)
// from(startScriptTilesProcessor)
// fileMode = 0b000_111_101_101 // 0755
// duplicatesStrategy = DuplicatesStrategy.EXCLUDE
//}
tasks.named("run") {
val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka/")
doFirst {
kafkaStateDir.asFile.mkdirs()
}
}
val runEventsProcessors by tasks.registering {
group = rootProject.name
dependsOn(tasks.run)
}

54
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/groupEntitiesMapChunks.kt Normal file
View file

@ -0,0 +1,54 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.core.launchTopology
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.colourEntityChunks
import dev.adamko.kafkatorio.server.processor.topology.convertEntityUpdateToServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.convertResourceUpdateToServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.KStream
/** Building and resource tiles */
suspend fun groupEntitiesMapChunks(
layer: ServerMapTileLayer,
colouredUpdatesStreamTopic: String,
colouredChunkStateTopic: String,
) {
val builder = StreamsBuilder()
val entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> =
when (layer) {
ServerMapTileLayer.TERRAIN -> return
ServerMapTileLayer.RESOURCE ->
builder.streamPacketData<MapChunkResourceUpdate>()
.convertResourceUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
ServerMapTileLayer.BUILDING ->
builder.streamPacketData<MapChunkEntityUpdate>()
.convertEntityUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
}
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
builder.streamPacketData()
val topology = colourEntityChunks(
builder,
entityChunksStream,
protosStream,
layer,
colouredUpdatesStreamTopic,
colouredChunkStateTopic,
)
launchTopology("groupTilesMapChunks.${layer.dir}", topology)
}

23
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/groupTilesMapChunks.kt Normal file
View file

@ -0,0 +1,23 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.core.launchTopology
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.KStream
/** Terrain tiles */
suspend fun groupTilesMapChunks() {
val builder = StreamsBuilder()
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
builder.streamPacketData()
val topology = colourMapChunks(builder, protosStream)
launchTopology("groupTilesMapChunks.${ServerMapTileLayer.TERRAIN.dir}", topology)
}

36
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/main.kt Normal file
View file

@ -0,0 +1,36 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
suspend fun main(): Unit = coroutineScope {
launch {
splitPackets()
}
launch {
groupTilesMapChunks()
}
launch {
groupEntitiesMapChunks(
layer = ServerMapTileLayer.BUILDING,
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES,
colouredChunkStateTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
)
}
launch {
groupEntitiesMapChunks(
layer = ServerMapTileLayer.RESOURCE,
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES,
colouredChunkStateTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
)
}
}

17
modules/events-processors/src/main/kotlin/dev/adamko/kafkatorio/processors/splitPackets.kt Normal file
View file

@ -0,0 +1,17 @@
package dev.adamko.kafkatorio.processors
import dev.adamko.kafkatorio.processor.core.launchTopology
import dev.adamko.kafkatorio.server.processor.topology.factorioServerPacketStream
import dev.adamko.kafkatorio.server.processor.topology.splitFactorioServerPacketStream
import org.apache.kafka.streams.StreamsBuilder
suspend fun splitPackets() {
val builder = StreamsBuilder()
val packets = factorioServerPacketStream(builder)
splitFactorioServerPacketStream(packets)
val topology = builder.build()
launchTopology("splitPackets", topology)
}

79
modules/events-server-core/build.gradle.kts
View file

@ -1,79 +0,0 @@
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
//description = "serves Kafkatorio content over the web"
val projectId: String by project.extra
dependencies {
implementation(platform(projects.modules.versionsPlatform))
implementation(libs.kafka.streams)
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
implementation(libs.kotlinx.coroutines.core)
implementation(libs.bundles.logging)
implementation(libs.scrimage.core)
implementation(libs.okio.core)
//<editor-fold desc="Ktor">
implementation(libs.ktorSerialization.kotlinxJson)
with(libs.ktorServer) {
implementation(auth)
implementation(authJwt)
implementation(core)
implementation(callId)
implementation(callLogging)
implementation(contentNegotiaion)
implementation(cio)
implementation(autoHeadResponse)
implementation(partialContent)
implementation(resources)
implementation(statusPages)
implementation(websockets)
implementation(compression)
}
with(libs.ktorNetwork) {
implementation(core)
implementation(tls)
}
//</editor-fold>
implementation(libs.simpleSyslogParser)
// implementation(libs.nimbusJoseJwt)
implementation(libs.kafka.kotkaStreams)
}
application {
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
"-opt-in=kotlinx.coroutines.FlowPreview",
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}

13
modules/events-server-syslog/build.gradle.kts
View file

@ -8,7 +8,7 @@ plugins {
}
//description = "serves Kafkatorio content over the web"
description = "Consumes Syslog Kafkatorio packets and sends them to Kafka"
val projectId: String by project.extra
@ -20,6 +20,7 @@ dependencies {
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(projects.modules.eventsProcessorCore)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
@ -62,11 +63,13 @@ dependencies {
// implementation(libs.nimbusJoseJwt)
implementation(libs.kafka.kotkaStreams)
implementation(libs.bundles.hoplite)
}
application {
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
mainClass.set("dev.adamko.kafkatorio.processor.syslog.SyslogMainKt")
}
@ -77,3 +80,9 @@ tasks.withType<KotlinCompile>().configureEach {
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}
val runSyslogServer by tasks.registering {
group = rootProject.name
dependsOn(tasks.run)
}

11
modules/events-server-syslog/docker/Dockerfile Normal file
View file

@ -0,0 +1,11 @@
FROM gradle:7-jdk11 AS build
COPY --chown=gradle:gradle . /home/gradle/src
WORKDIR /home/gradle/src
RUN gradle buildFatJar --no-daemon
FROM openjdk:11
EXPOSE 8080:8080
RUN mkdir /app
COPY --from=build /home/gradle/src/build/libs/*.jar /app/ktor-docker-sample.jar
ENTRYPOINT ["java","-jar","/app/ktor-docker-sample.jar"]

12
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/KafkatorioPacketKafkaProducer.kt → modules/events-server-syslog/src/main/kotlin/dev/adamko/kafkatorio/processor/syslog/KafkatorioPacketKafkaProducer.kt
View file

@ -1,12 +1,12 @@
package dev.adamko.kafkatorio.server.processor
package dev.adamko.kafkatorio.processor.syslog
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.core.Authenticator
import dev.adamko.kafkatorio.processor.syslog.config.jsonMapper
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.server.security.Authenticator
import dev.adamko.kafkatorio.server.socket.SyslogMsg
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
import dev.adamko.kafkatorio.processor.config.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.processor.config.TOPIC_SRC_SERVER_LOG_DLQ
import dev.adamko.kotka.kxs.kafkaSerializer
import java.util.Properties
import kotlin.coroutines.coroutineContext

2
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/socket/SyslogMsg.kt → modules/events-server-syslog/src/main/kotlin/dev/adamko/kafkatorio/processor/syslog/SyslogMsg.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.socket
package dev.adamko.kafkatorio.processor.syslog
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys

4
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/socket/SyslogSocketServer.kt → modules/events-server-syslog/src/main/kotlin/dev/adamko/kafkatorio/processor/syslog/SyslogSocketServer.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.server.socket
package dev.adamko.kafkatorio.processor.syslog
import com.github.palindromicity.syslog.AllowableDeviations
import com.github.palindromicity.syslog.DefaultKeyProvider
@ -7,7 +7,7 @@ import com.github.palindromicity.syslog.NilPolicy
import com.github.palindromicity.syslog.SyslogParser
import com.github.palindromicity.syslog.SyslogParserBuilder
import com.github.palindromicity.syslog.SyslogSpecification
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import io.ktor.network.selector.SelectorManager
import io.ktor.network.sockets.Socket
import io.ktor.network.sockets.aSocket

12
modules/events-server-syslog/src/main/kotlin/dev/adamko/kafkatorio/processor/syslog/config/kxsJson.kt Normal file
View file

@ -0,0 +1,12 @@
package dev.adamko.kafkatorio.processor.syslog.config
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
import kotlinx.serialization.json.Json
@PublishedApi
internal val jsonMapper: Json = Json {
prettyPrint = true
prettyPrintIndent = " "
serializersModule = jsonMapperKafkatorio.serializersModule
}

20
modules/events-server-syslog/src/main/kotlin/dev/adamko/kafkatorio/processor/syslog/syslogMain.kt Normal file
View file

@ -0,0 +1,20 @@
package dev.adamko.kafkatorio.processor.syslog
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.core.Authenticator
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
/** Receive log messages from the syslog socket, and send them to a Kafka topic */
suspend fun main(): Unit = coroutineScope {
val appProps = ApplicationProperties.load()
val syslogServer = SyslogSocketServer(appProps)
val authenticator = Authenticator(appProps)
val producer = KafkatorioPacketKafkaProducer(appProps, syslogServer, authenticator)
launch { syslogServer.start() }
launch { producer.launch() }
}

16
modules/events-server-api/build.gradle.kts → modules/events-server-web/build.gradle.kts
View file

@ -8,11 +8,11 @@ plugins {
}
//description = "serves Kafkatorio content over the web"
description = "serves Kafkatorio content over web APIs"
val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka")
dependencies {
@ -22,6 +22,7 @@ dependencies {
testImplementation(libs.kafka.streamsTestUtils)
implementation(projects.modules.eventsLibrary)
implementation(projects.modules.eventsProcessorCore)
implementation(libs.kotlinxSerialization.core)
implementation(libs.kotlinxSerialization.json)
@ -64,11 +65,13 @@ dependencies {
// implementation(libs.nimbusJoseJwt)
implementation(libs.kafka.kotkaStreams)
implementation(libs.bundles.hoplite)
}
application {
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
mainClass.set("dev.adamko.kafkatorio.server.web.MainKt")
}
@ -79,3 +82,10 @@ tasks.withType<KotlinCompile>().configureEach {
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}
val runEventsWebServer by tasks.registering {
group = rootProject.name
dependsOn(tasks.run)
}

4
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/config/kxsJson.kt → modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/config/kxsJson.kt
View file

@ -1,10 +1,10 @@
package dev.adamko.kafkatorio.server.config
package dev.adamko.kafkatorio.server.web.config
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
import kotlinx.serialization.json.Json
val jsonMapper: Json = Json {
internal val jsonMapper: Json = Json {
prettyPrint = true
prettyPrintIndent = " "
serializersModule = jsonMapperKafkatorio.serializersModule

61
modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/main.kt Normal file
View file

@ -0,0 +1,61 @@
package dev.adamko.kafkatorio.server.web
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.core.launchTopology
import dev.adamko.kafkatorio.server.processor.topology.broadcastToWebsocket
import dev.adamko.kafkatorio.server.processor.topology.saveMapTiles
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.seconds
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.StreamsConfig
suspend fun main(): Unit = coroutineScope {
val appProps = ApplicationProperties.load()
val wsServer = WebmapWebsocketServer()
launch {
websocketBroadcaster(wsServer)
}
// launch {
// broadcastToWebsocket(wsServer, StreamsBuilder())
// }
launch {
saveTiles(appProps)
}
launch {
startWebServer(
appProps = appProps,
wsServer = wsServer,
)
}
}
private suspend fun websocketBroadcaster(wsServer: WebmapWebsocketServer) {
val builder = StreamsBuilder()
broadcastToWebsocket(wsServer, builder)
val topology = builder.build()
launchTopology(
"websocketBroadcaster", topology, mapOf(
StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
)
)
}
private suspend fun saveTiles(
appProps: ApplicationProperties
) {
val builder = StreamsBuilder()
saveMapTiles(builder, appProps.serverDataDir)
launchTopology("saveTiles", builder.build())
}

6
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/factorioServersRoutes.kt → modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/factorioServersRoutes.kt
View file

@ -5,8 +5,8 @@ import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.config.jsonMapper
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.server.web.config.jsonMapper
import io.ktor.http.HttpStatusCode
import io.ktor.http.content.EntityTagVersion
import io.ktor.http.content.versions
@ -25,7 +25,7 @@ import kotlinx.serialization.Serializable
import kotlinx.serialization.json.encodeToJsonElement
fun Routing.factorioServersRoutes(
internal fun Routing.factorioServersRoutes(
appProps: ApplicationProperties,
) {

4
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/restApi.kt → modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/rest/restApi.kt
View file

@ -1,6 +1,6 @@
package dev.adamko.kafkatorio.server.web.rest
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.http.isSuccess
@ -22,7 +22,7 @@ import io.ktor.server.response.respondText
import io.ktor.server.routing.routing
fun Application.webmapTileServer(appProps: ApplicationProperties) {
internal fun Application.webmapTileServer(appProps: ApplicationProperties) {
install(AutoHeadResponse)
install(CallLogging) {

5
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/webServer.kt → modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/webServer.kt
View file

@ -1,6 +1,7 @@
package dev.adamko.kafkatorio.server.web
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.server.web.rest.webmapTileServer
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import dev.adamko.kafkatorio.server.web.websocket.webmapWebsocketServer
@ -9,7 +10,7 @@ import io.ktor.server.engine.embeddedServer
import kotlinx.coroutines.coroutineScope
suspend fun startWebServer(
internal suspend fun startWebServer(
appProps: ApplicationProperties,
wsServer: WebmapWebsocketServer,
): Unit = coroutineScope {

9
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/WebmapWebsocketServer.kt → modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/WebmapWebsocketServer.kt
View file

@ -1,5 +1,6 @@
package dev.adamko.kafkatorio.server.web.websocket
import dev.adamko.kafkatorio.server.processor.topology.WebsocketBroadcaster
import io.ktor.websocket.CloseReason
import io.ktor.websocket.DefaultWebSocketSession
import io.ktor.websocket.Frame
@ -24,7 +25,7 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.yield
class WebmapWebsocketServer {
class WebmapWebsocketServer : WebsocketBroadcaster {
private val startTime = TimeSource.Monotonic.markNow()
@ -88,9 +89,9 @@ class WebmapWebsocketServer {
}
suspend fun sendMessageToClients(msg: String) {
log("sending message to clients: $msg")
messagesToClients.emit(msg)
override suspend fun sendMessageToClients(message: String) {
log("sending message to clients: $message")
messagesToClients.emit(message)
}

0
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/wsServer.kt → modules/events-server-web/src/main/kotlin/dev/adamko/kafkatorio/server/web/websocket/wsServer.kt
View file

4
modules/events-server/build.gradle.kts
View file

@ -10,11 +10,11 @@ plugins {
}
description = "serves Kafkatorio content over the web"
description = "deprecated - this was split up into separate modules"
val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka")
dependencies {

2
modules/events-server/docker/Dockerfile
View file

@ -13,7 +13,7 @@ ENV SOCKET_SERVER_PORT=9002
EXPOSE $SOCKET_SERVER_PORT
ENV SERVER_DATA_DIR="/app/data"
ENV KAFKA_STREAMS_STATE_DIR="/app/kafka-state"
ENV KAFKA_STREAMS_STATE_DIR="/app/state/kafka"
VOLUME ${SERVER_DATA_DIR} \
${KAFKA_STREAMS_STATE_DIR}

56
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/eventsServer.kt
View file

@ -1,32 +1,32 @@
package dev.adamko.kafkatorio.server
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.processor.eventsProcessor
import dev.adamko.kafkatorio.server.security.Authenticator
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
import dev.adamko.kafkatorio.server.web.startWebServer
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
//import dev.adamko.kafkatorio.server.config.ApplicationProperties
//import dev.adamko.kafkatorio.server.processor.eventsProcessor
//import dev.adamko.kafkatorio.server.security.Authenticator
//import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
//import dev.adamko.kafkatorio.server.web.startWebServer
//import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
//import kotlinx.coroutines.coroutineScope
//import kotlinx.coroutines.launch
suspend fun main(): Unit = coroutineScope {
val appProps = ApplicationProperties.load()
val websocketServer = WebmapWebsocketServer()
val syslogSocketServer = SyslogSocketServer(appProps)
val authenticator = Authenticator(appProps)
launch {
startWebServer(appProps, websocketServer)
}
launch {
syslogSocketServer.start()
}
launch {
eventsProcessor(appProps, syslogSocketServer, websocketServer, authenticator)
}
}
//suspend fun main(): Unit = coroutineScope {
//
// val appProps = ApplicationProperties.load()
//
// val websocketServer = WebmapWebsocketServer()
// val syslogSocketServer = SyslogSocketServer(appProps)
// val authenticator = Authenticator(appProps)
//
// launch {
// startWebServer(appProps, websocketServer)
// }
//
// launch {
// syslogSocketServer.start()
// }
//
// launch {
// eventsProcessor(appProps, syslogSocketServer, websocketServer, authenticator)
// }
//}

432
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/KafkatorioTopology.kt
View file

@ -1,243 +1,243 @@
package dev.adamko.kafkatorio.server.processor
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.broadcastToWebsocket
import dev.adamko.kafkatorio.server.processor.topology.colourEntityChunks
import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
import dev.adamko.kafkatorio.server.processor.topology.convertEntityUpdateToServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.convertResourceUpdateToServerMapChunkTiles
import dev.adamko.kafkatorio.server.processor.topology.factorioServerPacketStream
import dev.adamko.kafkatorio.server.processor.topology.saveMapTiles
import dev.adamko.kafkatorio.server.processor.topology.splitFactorioServerPacketStream
import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import java.util.Properties
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
import kotlin.io.path.Path
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.seconds
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.cancel
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.streams.KafkaStreams
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.StreamsConfig
import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.TopologyDescription
import org.apache.kafka.streams.kstream.KStream
//import dev.adamko.kafkatorio.schema.common.FactorioServerId
//import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
//import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
//import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
//import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
//import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
//import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
//import dev.adamko.kafkatorio.server.config.ApplicationProperties
//import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
//import dev.adamko.kafkatorio.server.processor.topology.broadcastToWebsocket
//import dev.adamko.kafkatorio.server.processor.topology.colourEntityChunks
//import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
//import dev.adamko.kafkatorio.server.processor.topology.convertEntityUpdateToServerMapChunkTiles
//import dev.adamko.kafkatorio.server.processor.topology.convertResourceUpdateToServerMapChunkTiles
//import dev.adamko.kafkatorio.server.processor.topology.factorioServerPacketStream
//import dev.adamko.kafkatorio.server.processor.topology.saveMapTiles
//import dev.adamko.kafkatorio.server.processor.topology.splitFactorioServerPacketStream
//import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
//import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
//import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
//import java.util.Properties
//import kotlin.coroutines.resume
//import kotlin.coroutines.resumeWithException
//import kotlin.io.path.Path
//import kotlin.time.Duration.Companion.milliseconds
//import kotlin.time.Duration.Companion.seconds
//import kotlinx.coroutines.CoroutineName
//import kotlinx.coroutines.CoroutineScope
//import kotlinx.coroutines.cancel
//import kotlinx.coroutines.currentCoroutineContext
//import kotlinx.coroutines.job
//import kotlinx.coroutines.launch
//import kotlinx.coroutines.suspendCancellableCoroutine
//import org.apache.kafka.clients.producer.ProducerConfig
//import org.apache.kafka.streams.KafkaStreams
//import org.apache.kafka.streams.StreamsBuilder
//import org.apache.kafka.streams.StreamsConfig
//import org.apache.kafka.streams.Topology
//import org.apache.kafka.streams.TopologyDescription
//import org.apache.kafka.streams.kstream.KStream
internal class KafkatorioTopology(
private val appProps: ApplicationProperties,
private val wsServer: WebmapWebsocketServer,
private val syslogServer: SyslogSocketServer,
private val kafkatorioPacketKafkaProducer: KafkatorioPacketKafkaProducer,
// private val appProps: ApplicationProperties,
// private val wsServer: WebmapWebsocketServer,
// private val syslogServer: SyslogSocketServer,
// private val kafkatorioPacketKafkaProducer: KafkatorioPacketKafkaProducer,
) {
private val builderScope = CoroutineScope(CoroutineName("KafkaTopology"))
// private val builderScope = CoroutineScope(CoroutineName("KafkaTopology"))
suspend fun start() = builderScope.launch {
splitPackets()
groupTilesMapChunks()
groupEntitiesMapChunks(
layer = ServerMapTileLayer.BUILDING,
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES,
colouredChunkStateTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
)
groupEntitiesMapChunks(
layer = ServerMapTileLayer.RESOURCE,
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES,
colouredChunkStateTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
)
saveTiles()
websocketBroadcaster()
syslogProducer()
}
// suspend fun start() = builderScope.launch {
// splitPackets()
//
// groupTilesMapChunks()
//
//// groupEntitiesMapChunks(
//// layer = ServerMapTileLayer.BUILDING,
//// colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES,
//// colouredChunkStateTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
//// )
//
//// groupEntitiesMapChunks(
//// layer = ServerMapTileLayer.RESOURCE,
//// colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES,
//// colouredChunkStateTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
//// )
//
// saveTiles()
//
// websocketBroadcaster()
//
// syslogProducer()
// }
private suspend fun splitPackets() {
val builder = StreamsBuilder()
val packets = factorioServerPacketStream(builder)
splitFactorioServerPacketStream(packets)
val topology = builder.build()
launchTopology("splitPackets", topology)
}
// private suspend fun splitPackets() {
// val builder = StreamsBuilder()
// val packets = factorioServerPacketStream(builder)
// splitFactorioServerPacketStream(packets)
//
// val topology = builder.build()
//
// launchTopology("splitPackets", topology)
// }
/** Terrain tiles */
private suspend fun groupTilesMapChunks() {
val builder = StreamsBuilder()
// /** Terrain tiles */
// private suspend fun groupTilesMapChunks() {
// val builder = StreamsBuilder()
//
// val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
// builder.streamPacketData()
//
// val topology = colourMapChunks(builder, protosStream)
//
// launchTopology("groupTilesMapChunks.${ServerMapTileLayer.TERRAIN.dir}", topology)
// }
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
builder.streamPacketData()
val topology = colourMapChunks(builder, protosStream)
launchTopology("groupTilesMapChunks.${ServerMapTileLayer.TERRAIN.dir}", topology)
}
//
// /** Building and resource tiles */
// private suspend fun groupEntitiesMapChunks(
// layer: ServerMapTileLayer,
// colouredUpdatesStreamTopic: String,
// colouredChunkStateTopic: String,
// ) {
// val builder = StreamsBuilder()
//
// val entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> =
// when (layer) {
// ServerMapTileLayer.TERRAIN -> return
//
// ServerMapTileLayer.RESOURCE ->
// builder.streamPacketData<MapChunkResourceUpdate>()
// .convertResourceUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
//
// ServerMapTileLayer.BUILDING ->
// builder.streamPacketData<MapChunkEntityUpdate>()
// .convertEntityUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
// }
//
// val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
// builder.streamPacketData()
//
// val topology = colourEntityChunks(
// builder,
// entityChunksStream,
// protosStream,
// layer,
// colouredUpdatesStreamTopic,
// colouredChunkStateTopic,
// )
//
// launchTopology("groupTilesMapChunks.${layer.dir}", topology)
// }
/** Building and resource tiles */
private suspend fun groupEntitiesMapChunks(
layer: ServerMapTileLayer,
colouredUpdatesStreamTopic: String,
colouredChunkStateTopic: String,
) {
val builder = StreamsBuilder()
val entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> =
when (layer) {
ServerMapTileLayer.TERRAIN -> return
ServerMapTileLayer.RESOURCE ->
builder.streamPacketData<MapChunkResourceUpdate>()
.convertResourceUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
ServerMapTileLayer.BUILDING ->
builder.streamPacketData<MapChunkEntityUpdate>()
.convertEntityUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
}
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
builder.streamPacketData()
val topology = colourEntityChunks(
builder,
entityChunksStream,
protosStream,
layer,
colouredUpdatesStreamTopic,
colouredChunkStateTopic,
)
launchTopology("groupTilesMapChunks.${layer.dir}", topology)
}
// private suspend fun saveTiles() {
// val builder = StreamsBuilder()
// val topology = saveMapTiles(builder, appProps.serverDataDir)
// launchTopology("saveTiles", topology)
// }
private suspend fun saveTiles() {
val builder = StreamsBuilder()
val topology = saveMapTiles(builder, appProps.serverDataDir)
launchTopology("saveTiles", topology)
}
// private suspend fun websocketBroadcaster() {
// val builder = StreamsBuilder()
// broadcastToWebsocket(wsServer, builder)
//
// val topology = builder.build()
//
// launchTopology(
// "websocketBroadcaster", topology, mapOf(
// StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
// StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
// )
// )
// }
private suspend fun websocketBroadcaster() {
val builder = StreamsBuilder()
broadcastToWebsocket(wsServer, builder)
val topology = builder.build()
launchTopology(
"websocketBroadcaster", topology, mapOf(
StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
)
)
}
// /** Receive log messages from the syslog socket, and send them to a Kafka topic */
// private fun syslogProducer() {
// builderScope.launch {
// kafkatorioPacketKafkaProducer.launch()
// }
// }
/** Receive log messages from the syslog socket, and send them to a Kafka topic */
private fun syslogProducer() {
builderScope.launch {
kafkatorioPacketKafkaProducer.launch()
}
}
// private suspend fun launchTopology(
// id: String,
// topology: Topology,
// additionalProperties: Map<String, String> = mapOf(),
// ) {
//
// val props: Properties = appProps.kafkaStreamsConfig.toProperties()
//
// additionalProperties.forEach { (k, v) -> props.setProperty(k, v) }
//
// val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
// props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
//
// builderScope.launch(CoroutineName(appId)) {
//
// val streamsState: KafkaStreams.State = suspendCancellableCoroutine { cont ->
//
// val streams = KafkaStreams(topology, props)
// streams.setUncaughtExceptionHandler(StreamsExceptionHandler { e ->
// cont.resumeWithException(e)
// })
//
// streams.setStateListener { newState: KafkaStreams.State, _ ->
// when (newState) {
// KafkaStreams.State.NOT_RUNNING,
// KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
//
// else -> Unit // do nothing
// }
// }
//
// cont.invokeOnCancellation {
// println("[$appId] coroutine cancelled -> closing. cause:$it")
// streams.close()
// }
//
// printTopologyDescription(appId, topology)
// streams.start()
// println("launched Topology $appId")
// }
// println("exiting Topology $appId: $streamsState")
// currentCoroutineContext().job.cancel("$appId: $streamsState")
// }
// }
private suspend fun launchTopology(
id: String,
topology: Topology,
additionalProperties: Map<String, String> = mapOf(),
) {
val props: Properties = appProps.kafkaStreamsConfig.toProperties()
additionalProperties.forEach { (k, v) -> props.setProperty(k, v) }
val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
builderScope.launch(CoroutineName(appId)) {
val streamsState: KafkaStreams.State = suspendCancellableCoroutine { cont ->
val streams = KafkaStreams(topology, props)
streams.setUncaughtExceptionHandler(StreamsExceptionHandler { e ->
cont.resumeWithException(e)
})
streams.setStateListener { newState: KafkaStreams.State, _ ->
when (newState) {
KafkaStreams.State.NOT_RUNNING,
KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
else -> Unit // do nothing
}
}
cont.invokeOnCancellation {
println("[$appId] coroutine cancelled -> closing. cause:$it")
streams.close()
}
printTopologyDescription(appId, topology)
streams.start()
println("launched Topology $appId")
}
println("exiting Topology $appId: $streamsState")
currentCoroutineContext().job.cancel("$appId: $streamsState")
}
}
// private fun printTopologyDescription(appId: String, topology: Topology) {
// val description: TopologyDescription = topology.describe()
//
// val descFile = Path("./build/$appId.txt").toFile().apply {
// parentFile.mkdirs()
// if (!exists()) createNewFile()
// writeText(description.toString())
// }
//
// println(
// """
// |----------------
// |$appId - ${descFile.canonicalPath}
// |$description
// |----------------
// """.trimMargin()
// )
// }
private fun printTopologyDescription(appId: String, topology: Topology) {
val description: TopologyDescription = topology.describe()
val descFile = Path("./build/$appId.txt").toFile().apply {
parentFile.mkdirs()
if (!exists()) createNewFile()
writeText(description.toString())
}
println(
"""
|----------------
|$appId - ${descFile.canonicalPath}
|$description
|----------------
""".trimMargin()
)
}
companion object {
private fun Map<String, String>.toProperties(): Properties =
entries.fold(Properties()) { props, (k, v) ->
props.apply { setProperty(k, v) }
}
}
// companion object {
// private fun Map<String, String>.toProperties(): Properties =
// entries.fold(Properties()) { props, (k, v) ->
// props.apply { setProperty(k, v) }
// }
// }
}

64
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/eventProcessor.kt
View file

@ -1,35 +1,35 @@
package dev.adamko.kafkatorio.server.processor
import dev.adamko.kafkatorio.server.config.ApplicationProperties
import dev.adamko.kafkatorio.server.processor.admin.KafkatorioKafkaAdmin
import dev.adamko.kafkatorio.server.security.Authenticator
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
import kotlinx.coroutines.awaitCancellation
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.launch
suspend fun eventsProcessor(
appProps: ApplicationProperties,
syslogServer: SyslogSocketServer,
wsServer: WebmapWebsocketServer,
authenticator: Authenticator,
): Unit = coroutineScope {
val admin = KafkatorioKafkaAdmin(appProps)
admin.createKafkatorioTopics()
val kafkatorioPacketKafkaProducer = KafkatorioPacketKafkaProducer(appProps, syslogServer, authenticator)
val topology = KafkatorioTopology(appProps, wsServer, syslogServer, kafkatorioPacketKafkaProducer)
launch {
topology.start()
println("launched KafkatorioTopology")
awaitCancellation()
}
}
//import dev.adamko.kafkatorio.server.config.ApplicationProperties
//import dev.adamko.kafkatorio.server.processor.admin.KafkatorioKafkaAdmin
//import dev.adamko.kafkatorio.server.security.Authenticator
//import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
//import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
//import kotlinx.coroutines.awaitCancellation
//import kotlinx.coroutines.coroutineScope
//import kotlinx.coroutines.launch
//
//
//suspend fun eventsProcessor(
// appProps: ApplicationProperties,
// syslogServer: SyslogSocketServer,
// wsServer: WebmapWebsocketServer,
// authenticator: Authenticator,
//): Unit = coroutineScope {
//
// val admin = KafkatorioKafkaAdmin(appProps)
// admin.createKafkatorioTopics()
//
// val kafkatorioPacketKafkaProducer = KafkatorioPacketKafkaProducer(appProps, syslogServer, authenticator)
//
// val topology = KafkatorioTopology(appProps, wsServer, syslogServer, kafkatorioPacketKafkaProducer)
//
// launch {
// topology.start()
//
// println("launched KafkatorioTopology")
//
// awaitCancellation()
// }
//}

7
modules/infra-factorio-client/build.gradle.kts
View file

@ -109,3 +109,10 @@ tasks.processRun {
tasks.processKill {
dependsOn(clientKill)
}
val runFactorioClient by tasks.registering {
group = rootProject.name
dependsOn(tasks.processRun)
}

8
modules/infra-factorio-server/build.gradle.kts
View file

@ -35,7 +35,7 @@ val deployModToLocalServer by tasks.registering(Copy::class) {
into(factorioServerDataDir.dir("mods"))
doLast {
logger.lifecycle("Copying mods ${source.files} to $destinationDir")
logger.lifecycle("Copying mods ${source.files} into $destinationDir")
}
}
@ -75,3 +75,9 @@ idea {
excludeDirs.add(file("src/factorio-server"))
}
}
val runFactorioServer by tasks.registering {
group = rootProject.name
dependsOn(tasks.processRun)
}

2
modules/infra-factorio-server/src/docker-compose.yml
View file

@ -20,7 +20,7 @@ services:
driver: syslog
options:
syslog-address: "tcp://host.docker.internal:12034" # pc
#syslog-address: "tcp://localhost:12034" # nuc
# syslog-address: "tcp://localhost:12034" # nuc
syslog-format: "rfc5424"
tag: ${KAFKATORIO_TOKEN}

6
modules/infra-kafka-cluster/build.gradle.kts
View file

@ -27,3 +27,9 @@ tasks.dockerEnvUpdate {
// "SCHEMA_REGISTRY_MEM_LIMIT" to "256m",
)
}
val runKafkaCluster by tasks.registering {
group = rootProject.name
dependsOn(tasks.processRun)
}

4
modules/infra-kafka-cluster/src/.env
View file

@ -2,6 +2,6 @@
COMPOSE_PROJECT_NAME=kafkatorio
GRAFANA_VERSION=8.5.0
KAFKATORIO_VERSION=0.9.0
KAFKA_DL_URL=https\://dlcdn.apache.org/kafka/3.2.1/kafka_2.13-3.2.1.tgz
KAFKA_VERSION=3.2.1
KAFKA_DL_URL=https\://dlcdn.apache.org/kafka/3.2.2/kafka_2.13-3.2.2.tgz
KAFKA_VERSION=3.2.2
PROMETHEUS_VERSION=v2.34.0

36
modules/modules.md Normal file
View file

@ -0,0 +1,36 @@
```mermaid
flowchart TD
events-library-->events-processors & events-mod & web-map & events-server-web
kafka[(Kafka)]
events-mod-->infra-factorio-client
subgraph Kafka Workers
events-processor-core --> events-processor-syslog & events-processors
end
kafka<==>events-processors
subgraph infra
subgraph factorio
infra-factorio-client<-->infra-factorio-server
end
infra-factorio-server-->infra-kafka-pipe
infra-kafka-pipe-->kafka
infra-kafka-cluster-->kafka
end
kafka-->events-server-web
events-server-web-->web-map
versions-platform
subgraph experiments
kt-rcon
end
```

7
modules/web-map/build.gradle.kts
View file

@ -178,3 +178,10 @@ tasks.assemble {
tasks.withType<io.kvision.gradle.tasks.KVConvertPoTask>().configureEach {
enabled = false
}
val runWebMap by tasks.registering {
group = rootProject.name
dependsOn(tasks.matching { it.name == "browserDevelopmentRun" })
}

14
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/App.kt
View file

@ -58,13 +58,13 @@ object App : Application() {
require("./css/kafkatorio.css")
WebsocketService.packetsFlow
.filterIsInstance<EventServerPacket.ChunkTileSaved>()
.onEach { tileSavedEvent ->
// println("[packetsFlow] triggering tile refresh ${tileSavedEvent.filename.value}")
siteStateStore.dispatch(SiteAction.EventServerUpdate(tileSavedEvent))
// reduxStore.map.refreshUpdatedTilePng(tileSavedEvent.filename)
}.launchIn(coroutineScope)
// WebsocketService.packetsFlow
// .filterIsInstance<EventServerPacket.ChunkTileSaved>()
// .onEach { tileSavedEvent ->
//// println("[packetsFlow] triggering tile refresh ${tileSavedEvent.filename.value}")
//// siteStateStore.dispatch(SiteAction.EventServerUpdate(tileSavedEvent))
// siteStateStore.getState().factorioGameState?.map?.refreshUpdatedTilePng(tileSavedEvent.filename)
// }.launchIn(coroutineScope)
}

24
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/FactorioMap.kt
View file

@ -3,8 +3,10 @@ package dev.adamko.kafkatorio.webmap
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
import dev.adamko.kafkatorio.schema.packets.EventServerPacket
import dev.adamko.kafkatorio.webmap.externals.TileOnLoadFn
import dev.adamko.kafkatorio.webmap.externals.tileOnLoad
import dev.adamko.kafkatorio.webmap.services.WebsocketService
import io.kvision.maps.Maps
import io.kvision.maps.Maps.Companion.L
import io.kvision.maps.externals.leaflet.DoneCallback
@ -18,8 +20,15 @@ import io.kvision.maps.externals.leaflet.layer.tile.TileLayer
import io.kvision.utils.px
import kotlinx.browser.document
import kotlinx.browser.window
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.await
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.launch
import kotlinx.js.jso
import org.w3c.dom.HTMLElement
@ -36,6 +45,12 @@ class FactorioMap(
private val serverId: FactorioServerId,
) {
private val coroutineScope: CoroutineScope = CoroutineScope(
Dispatchers.Default
+ SupervisorJob(rootJob)
+ CoroutineName("FactorioMap")
)
val kvMap: Maps = Maps {
minHeight = 800.px
minWidth = 800.px
@ -90,6 +105,15 @@ class FactorioMap(
playerIconsLayer.addTo(this)
}
WebsocketService.packetsFlow
.filterIsInstance<EventServerPacket.ChunkTileSaved>()
.onEach { tileSavedEvent ->
// println("[packetsFlow] triggering tile refresh ${tileSavedEvent.filename.value}")
// siteStateStore.dispatch(SiteAction.EventServerUpdate(tileSavedEvent))
refreshUpdatedTilePng(tileSavedEvent.filename)
}.launchIn(coroutineScope)
}

27
settings.gradle.kts
View file

@ -1,18 +1,30 @@
rootProject.name = "kafkatorio"
@Suppress("UnstableApiUsage")
pluginManagement {
includeBuild("./gradle-plugins/settings-plugins/")
}
includeBuild("./gradle-plugins/builder-plugins/")
includeBuild("./gradle-plugins/factorio-mod-manager/")
include(
":modules:events-library",
":modules:events-mod",
":modules:events-processor-core",
":modules:events-processor-entities",
":modules:events-processor-packets",
":modules:events-processor-tiles",
// ":modules:events-processor-entities",
":modules:events-processors",
// ":modules:events-processor-tiles",
":modules:events-server",
":modules:events-server-api",
":modules:events-server-core",
// ":modules:events-server",
// ":modules:events-server-core",
":modules:events-server-web",
":modules:events-server-syslog",
":modules:infra-factorio-client",
@ -29,10 +41,13 @@ include(
":modules:versions-platform",
)
enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
apply(from = "./buildSrc/repositories.settings.gradle.kts")
@Suppress("UnstableApiUsage") // Central declaration of repositories is an incubating feature
dependencyResolutionManagement {
repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS)