big ol' refactor to split up the events-server and events-processors
This commit is contained in:
Adam
parent
d82f25c6ee
commit
873983f9a3
71 changed files with 1013 additions and 757 deletions
|
|
@ -4,15 +4,15 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
|
|||
plugins {
|
||||
dev.adamko.kafkatorio.lang.`kotlin-jvm`
|
||||
kotlin("plugin.serialization")
|
||||
application
|
||||
// application
|
||||
}
|
||||
|
||||
|
||||
//description = "serves Kafkatorio content over the web"
|
||||
description = "Shared Kafkatorio processors functionality"
|
||||
|
||||
val projectId: String by project.extra
|
||||
|
||||
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
|
||||
val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka")
|
||||
|
||||
|
||||
dependencies {
|
||||
|
|
@ -37,12 +37,17 @@ dependencies {
|
|||
implementation(libs.simpleSyslogParser)
|
||||
|
||||
implementation(libs.kafka.kotkaStreams)
|
||||
|
||||
implementation(libs.bundles.hoplite)
|
||||
|
||||
implementation(libs.ktorServer.authJwt)
|
||||
// implementation(libs.nimbusJoseJwt)
|
||||
}
|
||||
|
||||
|
||||
application {
|
||||
mainClass.set("dev.adamko.kafkatorio.server.EventsServerKt")
|
||||
}
|
||||
//application {
|
||||
// mainClass.set("dev.adamko.kafkatorio.processor.admin.MainKt")
|
||||
//}
|
||||
|
||||
|
||||
tasks.withType<KotlinCompile>().configureEach {
|
||||
|
|
@ -52,3 +57,10 @@ tasks.withType<KotlinCompile>().configureEach {
|
|||
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
val createKafkaTopics by tasks.registering(JavaExec::class) {
|
||||
group = rootProject.name
|
||||
classpath = sourceSets.main.get().runtimeClasspath
|
||||
mainClass.set("dev.adamko.kafkatorio.processor.admin.MainKt")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
package dev.adamko.kafkatorio.processor.admin
|
||||
|
||||
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.processor.config.allTopics
|
||||
import java.util.concurrent.TimeUnit
|
||||
import mu.KotlinLogging
|
||||
import org.apache.kafka.clients.admin.Admin
|
||||
import org.apache.kafka.clients.admin.CreateTopicsResult
|
||||
import org.apache.kafka.clients.admin.NewTopic
|
||||
import org.apache.kafka.clients.admin.TopicListing
|
||||
|
||||
|
||||
class KafkatorioKafkaAdmin(
|
||||
appProps: ApplicationProperties,
|
||||
) {
|
||||
|
||||
private val logger = KotlinLogging.logger {}
|
||||
|
||||
init {
|
||||
println("Creating KafkatorioKafkaAdmin with config: ${appProps.kafkaAdminConfig}")
|
||||
}
|
||||
|
||||
private val kafkaAdmin: Admin = Admin.create(appProps.kafkaAdminConfig)
|
||||
|
||||
fun createKafkatorioTopics() {
|
||||
|
||||
val currentTopics = currentTopics().map { it.name() }
|
||||
|
||||
val kafkatorioTopics = allTopics() - currentTopics.toSet()
|
||||
|
||||
if (kafkatorioTopics.isNotEmpty()) {
|
||||
logger.info("Creating ${kafkatorioTopics.size} topics")
|
||||
val result = kafkaAdmin.createTopics { kafkatorioTopics }
|
||||
|
||||
// wait for all brokers to become aware of the created topics
|
||||
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
||||
|
||||
result.values().forEach {
|
||||
logger.info("Created topic: ${it.key} ${it.value.get()}")
|
||||
}
|
||||
|
||||
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
||||
|
||||
logger.info("Listing topics")
|
||||
kafkaAdmin.listTopics()
|
||||
.listings()
|
||||
.get()
|
||||
.forEach {
|
||||
logger.info("Topic: $it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun Admin.createTopics(
|
||||
numPartitions: Int = 3,
|
||||
replicationFactor: Short = 1,
|
||||
topicNames: () -> Collection<String>,
|
||||
): CreateTopicsResult =
|
||||
createTopics(
|
||||
topicNames()
|
||||
.distinct()
|
||||
.map {
|
||||
NewTopic(it, numPartitions, replicationFactor)
|
||||
// .configs(mapOf("cleanup.policy" to "compact"))
|
||||
}
|
||||
)
|
||||
|
||||
private fun currentTopics(): MutableCollection<TopicListing> {
|
||||
return kafkaAdmin.listTopics()
|
||||
.listings()
|
||||
.get()
|
||||
}
|
||||
|
||||
// fun topicNameBase(packetType: KafkatorioPacketDataType): String =
|
||||
// "kafkatorio.${packetType.name}"
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
package dev.adamko.kafkatorio.processor.admin
|
||||
|
||||
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
|
||||
|
||||
|
||||
fun main() {
|
||||
val appProps = ApplicationProperties.load()
|
||||
val admin = KafkatorioKafkaAdmin(appProps)
|
||||
admin.createKafkatorioTopics()
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package dev.adamko.kafkatorio.processor.config
|
||||
|
||||
import com.sksamuel.hoplite.ConfigLoaderBuilder
|
||||
import com.sksamuel.hoplite.addEnvironmentSource
|
||||
import com.sksamuel.hoplite.addResourceOrFileSource
|
||||
import com.sksamuel.hoplite.sources.UserSettingsPropertySource
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerToken
|
||||
import dev.adamko.kafkatorio.schema.common.MaskedValue
|
||||
import java.nio.file.Path
|
||||
|
||||
val appProps = ApplicationProperties.load()
|
||||
|
||||
data class ApplicationProperties(
|
||||
val webPort: PortVal,
|
||||
val serverDataDir: Path,
|
||||
val kafkaAdminConfig: Map<String, String>,
|
||||
val kafkaStreamsConfig: Map<String, String>,
|
||||
|
||||
val socketServerHost: String,
|
||||
val socketServerPort: Int,
|
||||
|
||||
val jwtSecret: MaskedValue,
|
||||
|
||||
val kafkatorioServers: Map<FactorioServerToken, FactorioServerId>
|
||||
) {
|
||||
|
||||
companion object {
|
||||
/** use Hoplite to load config from environment variables or properties files */
|
||||
fun load(): ApplicationProperties =
|
||||
ConfigLoaderBuilder.default()
|
||||
.addDefaults()
|
||||
.allowUnresolvedSubstitutions()
|
||||
.allowEmptySources()
|
||||
.addEnvironmentSource()
|
||||
.addResourceOrFileSource(
|
||||
resourceOrFile = "${System.getProperty("user.home")}/.secret.kafkatorio-config.yml",
|
||||
optional = true
|
||||
)
|
||||
.addResourceOrFileSource("/.secret.config.yml", optional = true)
|
||||
.addResourceOrFileSource("/config.yml", optional = false)
|
||||
.addPropertySource(UserSettingsPropertySource)
|
||||
.build()
|
||||
.loadConfigOrThrow()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package dev.adamko.kafkatorio.processor.config
|
||||
|
||||
|
||||
/** Improved (slightly) version of http4k's `Port`. */
|
||||
@JvmInline
|
||||
value class PortVal(val value: Int) {
|
||||
|
||||
init {
|
||||
require(value in VALID_RANGE) {
|
||||
"Out of range. Port: '$value', range: $VALID_RANGE"
|
||||
}
|
||||
}
|
||||
|
||||
operator fun invoke(): Int = value
|
||||
|
||||
companion object {
|
||||
val DEFAULT = PortVal(8080)
|
||||
val VALID_RANGE: IntRange
|
||||
get() = (1..65535) // get() is required to prevent init ordering causing NPE
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
package dev.adamko.kafkatorio.processor.config
|
||||
|
||||
import dev.adamko.kafkatorio.schema.packets.ConfigurationUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.EntityUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.SurfaceUpdate
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
|
||||
private const val DOMAIN = "kafkatorio"
|
||||
|
||||
const val TOPIC_SRC_SERVER_LOG = "$DOMAIN.src.server-log"
|
||||
const val TOPIC_SRC_SERVER_LOG_DLQ = "$DOMAIN.src.server-log.dlq"
|
||||
|
||||
//const val TOPIC_GROUPED_MAP_CHUNKS_STATE = "$DOMAIN.state.map-chunks.grouped"
|
||||
|
||||
const val TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES =
|
||||
"$DOMAIN.state-updates.map-chunk-032.terrain.colour"
|
||||
const val TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES =
|
||||
"$DOMAIN.state-updates.map-chunk-032.resource.colour"
|
||||
const val TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES =
|
||||
"$DOMAIN.state-updates.map-chunk-032.building.colour"
|
||||
|
||||
const val TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE = "$DOMAIN.state.map-chunk.terrain.colour"
|
||||
const val TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE = "$DOMAIN.state.map-chunk.resource.colour"
|
||||
const val TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE = "$DOMAIN.state.map-chunk.building.colour"
|
||||
|
||||
//const val TOPIC_SUBDIVIDED_MAP_TILES = "$DOMAIN.map-tiles.subdivided"
|
||||
//const val TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED = "$TOPIC_SUBDIVIDED_MAP_TILES.debounced"
|
||||
|
||||
|
||||
/** Broadcast these events out, via websocket */
|
||||
const val TOPIC_BROADCAST_TO_WEBSOCKET = "$DOMAIN.broadcast.websocket"
|
||||
|
||||
fun allTopics(): Set<String> = buildSet {
|
||||
add(TOPIC_SRC_SERVER_LOG)
|
||||
add(TOPIC_SRC_SERVER_LOG_DLQ)
|
||||
|
||||
// add(TOPIC_GROUPED_MAP_CHUNKS_STATE)
|
||||
|
||||
add(TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES)
|
||||
add(TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES)
|
||||
add(TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES)
|
||||
|
||||
add(TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE)
|
||||
add(TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE)
|
||||
add(TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE)
|
||||
|
||||
// add(TOPIC_SUBDIVIDED_MAP_TILES)
|
||||
// add(TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED)
|
||||
|
||||
add(TOPIC_BROADCAST_TO_WEBSOCKET)
|
||||
|
||||
addAll(packetTopicNames.values)
|
||||
}
|
||||
|
||||
val KafkatorioPacketData.topicName: String
|
||||
get() = when (this) {
|
||||
is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class)
|
||||
is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class)
|
||||
is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class)
|
||||
is PrototypesUpdate -> packetTopicNames.getValue(PrototypesUpdate::class)
|
||||
is SurfaceUpdate -> packetTopicNames.getValue(SurfaceUpdate::class)
|
||||
is EntityUpdate -> packetTopicNames.getValue(EntityUpdate::class)
|
||||
is MapChunkTileUpdate -> packetTopicNames.getValue(MapChunkTileUpdate::class)
|
||||
is PlayerUpdate -> packetTopicNames.getValue(PlayerUpdate::class)
|
||||
is KafkatorioPacketDataError -> packetTopicNames.getValue(KafkatorioPacketDataError::class)
|
||||
is MapChunkEntityUpdate -> packetTopicNames.getValue(MapChunkEntityUpdate::class)
|
||||
is MapChunkResourceUpdate -> packetTopicNames.getValue(MapChunkResourceUpdate::class)
|
||||
}
|
||||
|
||||
|
||||
private val packetTopicNames: Map<KClass<out KafkatorioPacketData>, String> = mapOf(
|
||||
ConfigurationUpdate::class to "configuration",
|
||||
ConsoleChatUpdate::class to "console-chat",
|
||||
ConsoleCommandUpdate::class to "console-command",
|
||||
PrototypesUpdate::class to "prototypes",
|
||||
SurfaceUpdate::class to "surface",
|
||||
EntityUpdate::class to "entity",
|
||||
MapChunkTileUpdate::class to "map-chunk-tile",
|
||||
PlayerUpdate::class to "player",
|
||||
MapChunkEntityUpdate::class to "map-chunk-entity",
|
||||
MapChunkResourceUpdate::class to "map-chunk-resource",
|
||||
KafkatorioPacketDataError::class to "error",
|
||||
).mapValues { (_, v) ->
|
||||
"$DOMAIN.packet.$v"
|
||||
}
|
||||
|
||||
|
||||
val KClass<out KafkatorioPacketData>.topicName: String
|
||||
get() = packetTopicNames.getValue(this)
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
package dev.adamko.kafkatorio.processor.config
|
||||
|
||||
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
|
||||
import kotlinx.serialization.json.Json
|
||||
|
||||
|
||||
@PublishedApi
|
||||
internal val jsonMapper: Json = Json {
|
||||
prettyPrint = true
|
||||
prettyPrintIndent = " "
|
||||
serializersModule = jsonMapperKafkatorio.serializersModule
|
||||
}
|
||||
|
|
@ -0,0 +1,125 @@
|
|||
package dev.adamko.kafkatorio.processor.core
|
||||
|
||||
import com.auth0.jwt.JWT
|
||||
import com.auth0.jwt.algorithms.Algorithm
|
||||
import com.auth0.jwt.interfaces.DecodedJWT
|
||||
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerToken
|
||||
import kotlin.reflect.jvm.jvmName
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.serialization.SerialName
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import okio.ByteString.Companion.decodeBase64
|
||||
|
||||
|
||||
class Authenticator(
|
||||
appProps: ApplicationProperties,
|
||||
) {
|
||||
private val algorithm: Algorithm = Algorithm.HMAC256(appProps.jwtSecret.value)
|
||||
|
||||
private val verifier = JWT.require(algorithm).build()
|
||||
|
||||
fun verifyJwt(token: String): FactorioServerToken? {
|
||||
return runCatching {
|
||||
val decodedJwt: DecodedJWT? = verifier.verify(token)
|
||||
|
||||
val payloadJson = decodedJwt?.payload?.decodeBase64()?.string(Charsets.UTF_8) ?: "{}"
|
||||
|
||||
jsonMapper.decodeFromString(
|
||||
KafkatorioJwtPayload.serializer(),
|
||||
payloadJson,
|
||||
).serverToken
|
||||
|
||||
}.onFailure { e ->
|
||||
if (e is CancellationException) throw e
|
||||
|
||||
println("[Authenticator] Could not decode JWT ${e::class.jvmName}: ${e.message}")
|
||||
}.getOrNull()
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val jsonMapper = Json {
|
||||
ignoreUnknownKeys = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Serializable
|
||||
@SerialName("kafkatorio.auth.KafkatorioJwtPayload")
|
||||
private data class KafkatorioJwtPayload(
|
||||
@SerialName("sub")
|
||||
val serverToken: FactorioServerToken,
|
||||
/** unix epoch seconds*/
|
||||
@SerialName("iat")
|
||||
val issuedAtTime: Long,
|
||||
)
|
||||
|
||||
|
||||
//// https://github.com/auth0/java-jwt/blob/f799e589c3d43de318c7ca0f0c36e462297db973/lib/src/test/java/com/auth0/jwt/PemUtils.java
|
||||
//private object PemUtils {
|
||||
//
|
||||
// fun readPublicKey(filepath: String, algorithm: String): PublicKey {
|
||||
// val bytes = parsePemFile(File(filepath))
|
||||
// return getPublicKey(bytes, algorithm)
|
||||
// }
|
||||
//
|
||||
//// fun readPublicKey(reader: Reader, algorithm: String): PublicKey {
|
||||
//// return getPublicKey(bytes, algorithm)
|
||||
//// }
|
||||
//
|
||||
// fun readPrivateKey(filepath: String, algorithm: String): PrivateKey {
|
||||
// val bytes = parsePemFile(File(filepath))
|
||||
// return getPrivateKey(bytes, algorithm)
|
||||
// }
|
||||
//
|
||||
// private fun parsePemFile(pemFile: File): ByteArray {
|
||||
// return when {
|
||||
// !pemFile.isFile ->
|
||||
// throw FileNotFoundException("'${pemFile.absolutePath}' is not a file")
|
||||
//
|
||||
// !pemFile.exists() ->
|
||||
// throw FileNotFoundException("The file '${pemFile.absolutePath}' doesn't exist.")
|
||||
//
|
||||
// else ->
|
||||
// PemReader(FileReader(pemFile)).use { reader ->
|
||||
// reader.readPemObject().content
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// private fun getPublicKey(keyBytes: ByteArray, algorithm: String): PublicKey =
|
||||
// getKey(algorithm) {
|
||||
// val keySpec: EncodedKeySpec = X509EncodedKeySpec(keyBytes)
|
||||
// generatePublic(keySpec)
|
||||
// }
|
||||
//
|
||||
// private fun getPrivateKey(keyBytes: ByteArray, algorithm: String): PrivateKey =
|
||||
// getKey(algorithm) {
|
||||
// val keySpec: EncodedKeySpec = PKCS8EncodedKeySpec(keyBytes)
|
||||
// generatePrivate(keySpec)
|
||||
// }
|
||||
//
|
||||
//
|
||||
// private fun <K : Key> getKey(
|
||||
// algorithm: String,
|
||||
// generateKey: KeyFactory.() -> K
|
||||
// ): K = runCatching {
|
||||
// val kf = KeyFactory.getInstance(algorithm)
|
||||
// kf.generateKey()
|
||||
// }.getOrElse { e ->
|
||||
// when (e) {
|
||||
// is NoSuchAlgorithmException ->
|
||||
// throw IllegalStateException(
|
||||
// "Could not reconstruct key, algorithm '$algorithm' could not be found.",
|
||||
// e
|
||||
// )
|
||||
//
|
||||
// is InvalidKeySpecException ->
|
||||
// throw IllegalStateException("Could not reconstruct key", e)
|
||||
//
|
||||
// else -> throw e
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
package dev.adamko.kafkatorio.processor.core
|
||||
|
||||
import dev.adamko.kotka.extensions.component1
|
||||
import dev.adamko.kotka.extensions.component2
|
||||
import dev.adamko.kotka.extensions.component3
|
||||
import dev.adamko.kotka.extensions.processor.component1
|
||||
import dev.adamko.kotka.extensions.processor.component2
|
||||
import dev.adamko.kotka.extensions.processor.component3
|
||||
import dev.adamko.kotka.extensions.state.useAll
|
||||
import kotlin.time.toJavaDuration
|
||||
import org.apache.kafka.common.serialization.Serde
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.processor.Cancellable
|
||||
import org.apache.kafka.streams.processor.PunctuationType
|
||||
import org.apache.kafka.streams.processor.api.Processor
|
||||
import org.apache.kafka.streams.processor.api.ProcessorContext
|
||||
import org.apache.kafka.streams.processor.api.ProcessorSupplier
|
||||
import org.apache.kafka.streams.processor.api.Record
|
||||
import org.apache.kafka.streams.state.StoreBuilder
|
||||
import org.apache.kafka.streams.state.Stores
|
||||
import org.apache.kafka.streams.state.TimestampedKeyValueStore
|
||||
import org.apache.kafka.streams.state.ValueAndTimestamp
|
||||
import kotlin.time.Duration
|
||||
|
||||
|
||||
class DebounceProcessor<K, V>(
|
||||
private val inactivityDuration: Duration,
|
||||
private val checkInterval: Duration,
|
||||
private val storeName: String,
|
||||
) : Processor<K, V, K, V> {
|
||||
|
||||
private lateinit var state: State<K, V>
|
||||
|
||||
|
||||
override fun init(context: ProcessorContext<K, V>) {
|
||||
state = Active(context)
|
||||
}
|
||||
|
||||
|
||||
override fun process(record: Record<K, V>) {
|
||||
(state as Active).apply {
|
||||
val (key, value, _) = record
|
||||
|
||||
val existing: ValueAndTimestamp<V>? = store.get(key)
|
||||
val newTimestamp = existing?.timestamp() ?: System.currentTimeMillis()
|
||||
|
||||
store.put(key, ValueAndTimestamp.make(value, newTimestamp))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
override fun close() {
|
||||
(state as? Active)?.schedule?.cancel()
|
||||
state = Closed()
|
||||
}
|
||||
|
||||
|
||||
private sealed interface State<K, V>
|
||||
|
||||
|
||||
private class Closed<K, V> : State<K, V>
|
||||
|
||||
|
||||
private inner class Active(
|
||||
val context: ProcessorContext<K, V>
|
||||
) : State<K, V> {
|
||||
val store: TimestampedKeyValueStore<K, V> = context.getStateStore(storeName)
|
||||
val schedule: Cancellable = context.schedule(
|
||||
checkInterval.toJavaDuration(),
|
||||
PunctuationType.WALL_CLOCK_TIME,
|
||||
::checkRecords
|
||||
)
|
||||
|
||||
private fun checkRecords(now: Long) {
|
||||
store.useAll {
|
||||
forEach { record ->
|
||||
val (key, value, timestamp) = record
|
||||
if ((now - timestamp) > inactivityDuration.inWholeMilliseconds) {
|
||||
context.forward(Record(key, value, timestamp))
|
||||
store.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
|
||||
fun <K, V> Topology.addDebounceProcessor(
|
||||
namePrefix: String,
|
||||
sourceTopic: String,
|
||||
sinkTopic: String,
|
||||
|
||||
inactivityDuration: Duration,
|
||||
checkInterval: Duration = inactivityDuration / 10,
|
||||
keySerde: Serde<K>,
|
||||
valueSerde: Serde<V>,
|
||||
storeName: String = "$namePrefix.debounce-store",
|
||||
): Topology {
|
||||
|
||||
val sourceName = "$namePrefix.debounce-source"
|
||||
val processorName = "$namePrefix.debounce-processor"
|
||||
val sinkName = "$namePrefix.debounce-sink"
|
||||
|
||||
return addSource(
|
||||
sourceName,
|
||||
keySerde.deserializer(),
|
||||
valueSerde.deserializer(),
|
||||
sourceTopic,
|
||||
).addProcessor(
|
||||
processorName,
|
||||
Supplier(
|
||||
inactivityDuration = inactivityDuration,
|
||||
checkInterval = checkInterval,
|
||||
storeName = storeName,
|
||||
keySerde = keySerde,
|
||||
valueSerde = valueSerde,
|
||||
),
|
||||
sourceName,
|
||||
).addSink(
|
||||
sinkName,
|
||||
sinkTopic,
|
||||
keySerde.serializer(),
|
||||
valueSerde.serializer(),
|
||||
processorName,
|
||||
)
|
||||
}
|
||||
|
||||
class Supplier<K, V>(
|
||||
private val inactivityDuration: Duration,
|
||||
private val checkInterval: Duration,
|
||||
private val storeName: String,
|
||||
private val keySerde: Serde<K>,
|
||||
private val valueSerde: Serde<V>,
|
||||
) : ProcessorSupplier<K, V, K, V> {
|
||||
|
||||
override fun get(): Processor<K, V, K, V> =
|
||||
DebounceProcessor(inactivityDuration, checkInterval, storeName)
|
||||
|
||||
override fun stores(): Set<StoreBuilder<*>> = setOf(
|
||||
Stores.timestampedKeyValueStoreBuilder(
|
||||
Stores.persistentTimestampedKeyValueStore(storeName),
|
||||
keySerde,
|
||||
valueSerde,
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package dev.adamko.kafkatorio.processor.core
|
||||
|
||||
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler
|
||||
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse
|
||||
|
||||
|
||||
class StreamsExceptionHandler(
|
||||
val shutdown: (e: Throwable) -> Unit,
|
||||
) : StreamsUncaughtExceptionHandler {
|
||||
override fun handle(exception: Throwable): StreamThreadExceptionResponse {
|
||||
exception.printStackTrace()
|
||||
shutdown(exception)
|
||||
return StreamThreadExceptionResponse.SHUTDOWN_APPLICATION
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
package dev.adamko.kafkatorio.processor.core
|
||||
|
||||
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
|
||||
import java.util.Properties
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.resumeWithException
|
||||
import kotlin.io.path.Path
|
||||
import kotlinx.coroutines.CoroutineName
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.job
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import org.apache.kafka.streams.KafkaStreams
|
||||
import org.apache.kafka.streams.StreamsConfig
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.TopologyDescription
|
||||
|
||||
|
||||
private val appProps = ApplicationProperties.load()
|
||||
|
||||
suspend fun launchTopology(
|
||||
id: String,
|
||||
topology: Topology,
|
||||
additionalProperties: Map<String, String> = mapOf(),
|
||||
): Unit = coroutineScope {
|
||||
|
||||
val props: Properties = appProps.kafkaStreamsConfig.toProperties()
|
||||
|
||||
additionalProperties.forEach { (k, v) -> props.setProperty(k, v) }
|
||||
|
||||
val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
|
||||
props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
|
||||
|
||||
launch(CoroutineName(appId)) {
|
||||
|
||||
val streamsState: KafkaStreams.State = suspendCancellableCoroutine { cont ->
|
||||
|
||||
val streams = KafkaStreams(topology, props)
|
||||
streams.setUncaughtExceptionHandler(StreamsExceptionHandler { e ->
|
||||
cont.resumeWithException(e)
|
||||
})
|
||||
|
||||
streams.setStateListener { newState: KafkaStreams.State, _ ->
|
||||
when (newState) {
|
||||
KafkaStreams.State.PENDING_SHUTDOWN,
|
||||
KafkaStreams.State.NOT_RUNNING,
|
||||
KafkaStreams.State.PENDING_ERROR,
|
||||
KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
|
||||
|
||||
else -> Unit // do nothing
|
||||
}
|
||||
}
|
||||
|
||||
cont.invokeOnCancellation {
|
||||
println("[$appId] coroutine cancelled -> closing. cause:$it")
|
||||
streams.close()
|
||||
}
|
||||
|
||||
printTopologyDescription(appId, topology)
|
||||
streams.start()
|
||||
println("launched Topology $appId")
|
||||
}
|
||||
println("exiting Topology $appId: $streamsState")
|
||||
currentCoroutineContext().job.cancel("$appId: $streamsState")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun printTopologyDescription(appId: String, topology: Topology) {
|
||||
val description: TopologyDescription = topology.describe()
|
||||
|
||||
val descFile = Path("./build/$appId.txt").toFile().apply {
|
||||
parentFile.mkdirs()
|
||||
if (!exists()) createNewFile()
|
||||
writeText(description.toString())
|
||||
}
|
||||
|
||||
println(
|
||||
"""
|
||||
|----------------
|
||||
|$appId - ${descFile.canonicalPath}
|
||||
|$description
|
||||
|----------------
|
||||
""".trimMargin()
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,329 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.library.resize
|
||||
import dev.adamko.kafkatorio.library.toMapChunkPosition
|
||||
import dev.adamko.kafkatorio.processor.core.DebounceProcessor.Companion.addDebounceProcessor
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.groupedAs
|
||||
import dev.adamko.kotka.extensions.materializedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.repartitionedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.flatMap
|
||||
import dev.adamko.kotka.extensions.streams.map
|
||||
import dev.adamko.kotka.extensions.streams.peek
|
||||
import dev.adamko.kotka.extensions.streams.reduce
|
||||
import dev.adamko.kotka.extensions.tableJoined
|
||||
import dev.adamko.kotka.extensions.tables.join
|
||||
import dev.adamko.kotka.extensions.tables.toStream
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
import org.apache.kafka.streams.kstream.KTable
|
||||
|
||||
|
||||
/** Add colour to each [MapChunkTileUpdate] */
|
||||
fun colourMapChunks(
|
||||
builder: StreamsBuilder,
|
||||
protosStream: KStream<FactorioServerId, PrototypesUpdate>
|
||||
): Topology {
|
||||
val pid = "colourMapChunks.terrain"
|
||||
|
||||
val mapChunksStream: KStream<FactorioServerId, MapChunkTileUpdate> = builder.streamPacketData()
|
||||
|
||||
val protosTable: KTable<FactorioServerId, TileColourDict> =
|
||||
protosStream.createTilePrototypeTable()
|
||||
|
||||
val chunkTilesColouredTable032: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
mapChunksStream
|
||||
.convertToServerMapChunkTiles(pid)
|
||||
.reduceServerMapChunkTilesToTable(pid)
|
||||
.enrichWithColourData(pid, protosTable)
|
||||
|
||||
chunkTilesColouredTable032.streamMapChunkColouredTo(
|
||||
"$pid.${ChunkSize.CHUNK_032}",
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES,
|
||||
)
|
||||
|
||||
val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
builder.stream(
|
||||
consumedAs(
|
||||
"$pid.consume.map-chunks-debounced",
|
||||
kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
),
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
|
||||
)
|
||||
|
||||
mapOf(
|
||||
// ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
|
||||
// ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
|
||||
// ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
|
||||
// ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
|
||||
ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
|
||||
ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
).forEach { (from, to) ->
|
||||
chunkTilesColoured
|
||||
.filter("$pid.combine-chunks.filter-$from-to-$to") { key: ServerMapChunkId, _: ServerMapChunkTiles<ColourHex> ->
|
||||
key.chunkSize == from
|
||||
}.groupByChunkPosition("$pid.from-$from-to-$to", to)
|
||||
.streamMapChunkColouredTo(
|
||||
"$pid.from-$from-to-$to",
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
|
||||
)
|
||||
}
|
||||
|
||||
return builder.build()
|
||||
.addDebounceProcessor(
|
||||
namePrefix = "groupMapChunks.terrain",
|
||||
sourceTopic = TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES,
|
||||
sinkTopic = TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
|
||||
inactivityDuration = 15.seconds,
|
||||
keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
private fun KStream<FactorioServerId, MapChunkTileUpdate>.convertToServerMapChunkTiles(
|
||||
callerPid: String
|
||||
): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.convertToMapTiles"
|
||||
|
||||
return map("$pid.map") { serverId: FactorioServerId, update: MapChunkTileUpdate ->
|
||||
|
||||
val mapTileList = update.tileDictionary?.toMapTileList() ?: emptyList()
|
||||
|
||||
val (validMapTiles, invalidMapTiles) = mapTileList.partition {
|
||||
it.position in update.key.chunkPosition
|
||||
}
|
||||
|
||||
if (invalidMapTiles.isNotEmpty()) {
|
||||
println(
|
||||
"WARNING [$pid] MapChunkTileUpdate contained ${invalidMapTiles.size} out-of-bounds tiles, " +
|
||||
"key: ${update.key},\n" +
|
||||
"invalid tiles:" + invalidMapTiles.joinToString(limit = 10)
|
||||
)
|
||||
}
|
||||
|
||||
val validMapTilesToProtoHashCode = validMapTiles.associate {
|
||||
it.position to PrototypeHashCode(it.protoId)
|
||||
}
|
||||
|
||||
val chunkId = ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.TERRAIN,
|
||||
chunkPosition = update.key.chunkPosition,
|
||||
surfaceIndex = update.key.surfaceIndex,
|
||||
)
|
||||
|
||||
chunkId to ServerMapChunkTiles(chunkId, validMapTilesToProtoHashCode)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.reduceServerMapChunkTilesToTable(
|
||||
callerPid: String,
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.reduceServerMapChunkTilesToTable"
|
||||
|
||||
return repartition(
|
||||
repartitionedAs(
|
||||
"$pid.pre-table-repartition",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
|
||||
// force, otherwise KTable-KTable FK join doesn't work
|
||||
numberOfPartitions = 1,
|
||||
)
|
||||
).groupByKey(
|
||||
groupedAs(
|
||||
"$pid.group-by-key",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
|
||||
)
|
||||
).reduce(
|
||||
"$pid.reduce",
|
||||
materializedAs(
|
||||
"$pid.reduce.store",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
|
||||
)
|
||||
) { chunkTiles, otherChunkTiles ->
|
||||
chunkTiles + otherChunkTiles
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.enrichWithColourData(
|
||||
callerPid: String,
|
||||
protosTable: KTable<FactorioServerId, TileColourDict>,
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val pid = "$callerPid.enrichWithColourData"
|
||||
|
||||
return join(
|
||||
other = protosTable,
|
||||
tableJoined = tableJoined("$pid.join-tiles-with-prototypes"),
|
||||
materialized = materializedAs(
|
||||
"$pid.join-tiles-with-prototypes.store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
),
|
||||
foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<PrototypeHashCode> ->
|
||||
chunkTiles.chunkId.serverId
|
||||
}
|
||||
) { tiles: ServerMapChunkTiles<PrototypeHashCode>, colourDict: TileColourDict ->
|
||||
println("joining tiles:${tiles.map.size} with colourDict:${colourDict.size}")
|
||||
|
||||
val missingProtos = mutableSetOf<PrototypeHashCode>()
|
||||
|
||||
val tileColours: Map<MapTilePosition, ColourHex> =
|
||||
tiles.map.mapNotNull { (position, code) ->
|
||||
when (val colour = colourDict[code]) {
|
||||
null -> {
|
||||
missingProtos.add(code)
|
||||
null
|
||||
}
|
||||
else -> position to colour
|
||||
}
|
||||
}.toMap()
|
||||
|
||||
if (missingProtos.isNotEmpty()) {
|
||||
println("[$pid] missing ${missingProtos.size} tile prototypes: ${missingProtos.joinToString { "${it.code}" }}")
|
||||
}
|
||||
|
||||
println("[$pid] Set ${tileColours.size} tile colours for chunk ${tiles.chunkId}")
|
||||
|
||||
ServerMapChunkTiles(tiles.chunkId, tileColours)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.streamMapChunkColouredTo(
|
||||
callerPid: String,
|
||||
outputTopic: String,
|
||||
) {
|
||||
val pid = "$callerPid.output-chunk"
|
||||
|
||||
toStream("$pid.stream")
|
||||
.flatMap("$pid.group-by-chunks") { _: ServerMapChunkId, tiles: ServerMapChunkTiles<ColourHex>? ->
|
||||
reChunkTiles(pid, tiles)
|
||||
}.peek("$pid.print-group-result") { _, chunkTiles ->
|
||||
println("[$pid] Grouping map tiles result: ${chunkTiles.chunkId} / size:${chunkTiles.map.size}")
|
||||
}.to(
|
||||
outputTopic,
|
||||
producedAs(
|
||||
"$pid.grouped-map-chunks",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Group [MapChunkTileUpdate]s by the Chunk position.
|
||||
*
|
||||
* They should already be grouped, but do it again to make sure, and to filter out empty updates.
|
||||
*/
|
||||
fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.groupByChunkPosition(
|
||||
callerPid: String,
|
||||
targetChunkSize: ChunkSize = ChunkSize.CHUNK_032
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val pid = "$callerPid.groupByChunkPosition.${targetChunkSize.name}"
|
||||
|
||||
return flatMap("$pid.change-chunk-size") { _: ServerMapChunkId, chunkTiles: ServerMapChunkTiles<ColourHex>? ->
|
||||
// println("[$pid] changing chunk size for $chunkId")
|
||||
chunkTiles ?: return@flatMap emptyList()
|
||||
|
||||
val newChunkPosition = chunkTiles.chunkId.chunkPosition.resize(targetChunkSize)
|
||||
val newId = chunkTiles.chunkId.copy(chunkPosition = newChunkPosition)
|
||||
val newChunkTiles = chunkTiles.copy(chunkId = newId)
|
||||
|
||||
reChunkTiles(pid, newChunkTiles)
|
||||
}.groupByKey(
|
||||
groupedAs(
|
||||
"$pid.group-by-key",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
|
||||
)
|
||||
).reduce(
|
||||
"$pid.reduce",
|
||||
materializedAs(
|
||||
"$pid.reduce.store",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
|
||||
)
|
||||
) { chunkTiles, otherChunkTiles ->
|
||||
chunkTiles + otherChunkTiles
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun <T : Any> reChunkTiles(
|
||||
pid: String,
|
||||
tiles: ServerMapChunkTiles<T>?,
|
||||
): List<Pair<ServerMapChunkId, ServerMapChunkTiles<T>>> {
|
||||
|
||||
return when {
|
||||
tiles == null || tiles.map.isEmpty() -> emptyList()
|
||||
|
||||
tiles.map.keys.all { it in tiles.chunkId.chunkPosition } -> listOf(tiles.chunkId to tiles)
|
||||
|
||||
else -> {
|
||||
val chunkId = tiles.chunkId
|
||||
|
||||
val (boundedTiles, unboundedTiles) = tiles.map.entries.partition { (position, _) ->
|
||||
position in chunkId.chunkPosition
|
||||
}
|
||||
|
||||
if (unboundedTiles.isNotEmpty()) {
|
||||
val unboundedEntities = unboundedTiles.map { (_, entity) -> entity }
|
||||
println(
|
||||
"WARNING [$pid] MapChunkEntityUpdate $chunkId "
|
||||
+ "contained ${unboundedEntities.size} out-of-bounds entities "
|
||||
+ unboundedEntities.joinToString(limit = 10)
|
||||
)
|
||||
}
|
||||
|
||||
// the existing chunkId and its tiles
|
||||
val boundedTilesChunk =
|
||||
chunkId to ServerMapChunkTiles(chunkId, boundedTiles.associate { it.key to it.value })
|
||||
|
||||
// re-calculate a chunk for the out-of-bounds tiles
|
||||
val chunkedUnboundedTiles = unboundedTiles
|
||||
.map { it.key to it.value }
|
||||
.groupBy { (position, _) ->
|
||||
position.toMapChunkPosition(chunkId.chunkSize)
|
||||
}.map { (newChunkPosition, tiles) ->
|
||||
val newChunkId = chunkId.copy(chunkPosition = newChunkPosition)
|
||||
newChunkId to ServerMapChunkTiles(newChunkId, tiles.toMap())
|
||||
}
|
||||
|
||||
println("[$pid] re-bounded ${unboundedTiles.size} tiles into ${chunkedUnboundedTiles.size} chunks in $chunkId")
|
||||
|
||||
listOf(boundedTilesChunk) + chunkedUnboundedTiles
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,272 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.library.toMapTilePosition
|
||||
import dev.adamko.kafkatorio.processor.core.DebounceProcessor.Companion.addDebounceProcessor
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioPrototype
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapBoundingBox
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.common.toHex
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.materializedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.flatMap
|
||||
import dev.adamko.kotka.extensions.tableJoined
|
||||
import dev.adamko.kotka.extensions.tables.join
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.math.floor
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
import org.apache.kafka.streams.kstream.KTable
|
||||
|
||||
|
||||
/**
|
||||
* Add colour to each [MapChunkTileUpdate]
|
||||
*
|
||||
* @param[colouredUpdatesStreamTopic] Output topic for the result of converting input updates into 32 sized chunks
|
||||
* @param[colouredChunkStateTopic] Post-debounce state topic, used to recursively combine chunks
|
||||
*/
|
||||
fun colourEntityChunks(
|
||||
builder: StreamsBuilder,
|
||||
entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>,
|
||||
protosStream: KStream<FactorioServerId, PrototypesUpdate>,
|
||||
layer: ServerMapTileLayer,
|
||||
colouredUpdatesStreamTopic: String,
|
||||
colouredChunkStateTopic: String,
|
||||
): Topology {
|
||||
val pid = "colourEntityChunks.${layer.dir}"
|
||||
|
||||
val protosTable: KTable<FactorioServerId, EntityPrototypeMap> =
|
||||
protosStream.createEntityPrototypeMap(pid)
|
||||
|
||||
val chunkEntitiesColouredTable032: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
entityChunksStream
|
||||
.reduceServerMapChunkTilesToTable(pid)
|
||||
.enrichWithColourData(pid, protosTable)
|
||||
|
||||
chunkEntitiesColouredTable032.streamMapChunkColouredTo(
|
||||
"$pid.${ChunkSize.CHUNK_032}",
|
||||
colouredUpdatesStreamTopic,
|
||||
)
|
||||
|
||||
val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
builder.stream(
|
||||
consumedAs(
|
||||
"$pid.consume.map-chunks-debounced",
|
||||
kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
),
|
||||
colouredChunkStateTopic,
|
||||
)
|
||||
|
||||
mapOf(
|
||||
// ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
|
||||
// ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
|
||||
// ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
|
||||
// ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
|
||||
ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
|
||||
ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
).forEach { (from, to) ->
|
||||
chunkTilesColoured
|
||||
.filter("$pid.combine-chunks.filter-$from-to-$to") { key, _ ->
|
||||
key.chunkSize == from && key.layer == layer
|
||||
}.groupByChunkPosition(pid, to)
|
||||
.streamMapChunkColouredTo(
|
||||
"$pid.from-$from-to-$to",
|
||||
colouredChunkStateTopic,
|
||||
)
|
||||
}
|
||||
|
||||
return builder.build().addDebounceProcessor(
|
||||
namePrefix = "groupMapChunks.${layer.dir}",
|
||||
sourceTopic = colouredUpdatesStreamTopic,
|
||||
sinkTopic = colouredChunkStateTopic,
|
||||
inactivityDuration = 15.seconds,
|
||||
keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
fun KStream<FactorioServerId, MapChunkEntityUpdate>.convertEntityUpdateToServerMapChunkTiles(
|
||||
callerPid: String,
|
||||
): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.convertToServerMapChunkTiles"
|
||||
|
||||
return flatMap("$pid.map") { serverId: FactorioServerId, update: MapChunkEntityUpdate ->
|
||||
|
||||
//
|
||||
// // TODO There might be multiple resources per tile position, therefore this will arbitrarily
|
||||
// // exclude resources. Maybe change it from a Map to a List? Or have multiple resources per
|
||||
// // tile position?
|
||||
// val mapValidEntitiesToPrototypeHash = validEntityPositions.associate { (pos, resource) ->
|
||||
// pos to PrototypeHashCode(resource.protoId)
|
||||
// }
|
||||
|
||||
val entitiesToPrototypeHash = update.entities().associate { entity ->
|
||||
entity.position.toMapTilePosition() to PrototypeHashCode(entity.protoId)
|
||||
}
|
||||
|
||||
val chunkId = ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.BUILDING,
|
||||
chunkPosition = update.key.chunkPosition,
|
||||
surfaceIndex = update.key.surfaceIndex,
|
||||
)
|
||||
|
||||
val tiles = ServerMapChunkTiles(chunkId, entitiesToPrototypeHash)
|
||||
|
||||
reChunkTiles(pid, tiles)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun KStream<FactorioServerId, MapChunkResourceUpdate>.convertResourceUpdateToServerMapChunkTiles(
|
||||
callerPid: String,
|
||||
): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.convertResourceUpdateToChunkTiles"
|
||||
|
||||
return flatMap("$pid.map") { serverId: FactorioServerId, update: MapChunkResourceUpdate ->
|
||||
|
||||
// // TODO There might be multiple resources per tile position, therefore this will arbitrarily
|
||||
// // exclude resources. Maybe change it from a Map to a List? Or have multiple resources per
|
||||
// // tile position?
|
||||
// val mapValidEntitiesToPrototypeHash = validEntityPositions.associate { (pos, resource) ->
|
||||
// pos to PrototypeHashCode(resource.protoId)
|
||||
// }
|
||||
|
||||
val entitiesToPrototypeHash = update.resources().associate { entity ->
|
||||
entity.position.toMapTilePosition() to PrototypeHashCode(entity.protoId)
|
||||
}
|
||||
|
||||
val chunkId = ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.RESOURCE,
|
||||
chunkPosition = update.key.chunkPosition,
|
||||
surfaceIndex = update.key.surfaceIndex,
|
||||
)
|
||||
|
||||
val tiles = ServerMapChunkTiles(chunkId, entitiesToPrototypeHash)
|
||||
|
||||
reChunkTiles(pid, tiles)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.enrichWithColourData(
|
||||
callerPid: String,
|
||||
protosTable: KTable<FactorioServerId, EntityPrototypeMap>,
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val pid = "$callerPid.enrichWithColourData"
|
||||
|
||||
return join(
|
||||
other = protosTable,
|
||||
tableJoined = tableJoined("$pid.join-tiles-with-prototypes"),
|
||||
materialized = materializedAs(
|
||||
"$pid.join-tiles-with-prototypes.store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
),
|
||||
foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<PrototypeHashCode> ->
|
||||
chunkTiles.chunkId.serverId
|
||||
}) { tiles: ServerMapChunkTiles<PrototypeHashCode>, prototypes: EntityPrototypeMap ->
|
||||
println("joining tiles:${tiles.map.size} with prototypes:${prototypes.size}")
|
||||
|
||||
val missingProtos = mutableSetOf<PrototypeHashCode>()
|
||||
|
||||
val tileColours: Map<MapTilePosition, ColourHex> = buildList {
|
||||
tiles.map.forEach { (position, code) ->
|
||||
when (val proto = prototypes[code]) {
|
||||
|
||||
null -> missingProtos += code
|
||||
|
||||
else -> when (val box = proto.collisionBox) {
|
||||
|
||||
null -> add(position to proto.mapColour())
|
||||
|
||||
else -> box.iterator(position).forEach { entitySubTile ->
|
||||
add(entitySubTile to proto.mapColour())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}.fold(mapOf()) { acc, (pos, colour) ->
|
||||
// merge overlapping colours
|
||||
val newColour = when (val existing = acc[pos]) {
|
||||
null -> colour
|
||||
else -> (existing / 2u) + (colour / 2u)
|
||||
}
|
||||
acc + (pos to newColour)
|
||||
}
|
||||
|
||||
if (missingProtos.isNotEmpty()) {
|
||||
println("[$pid] missing ${missingProtos.size} tile prototypes: ${missingProtos.joinToString { "${it.code}" }}")
|
||||
}
|
||||
|
||||
println("[$pid] Set ${tileColours.size} tile colours for chunk ${tiles.chunkId}")
|
||||
|
||||
ServerMapChunkTiles(tiles.chunkId, tileColours)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun MapBoundingBox.iterator(origin: MapTilePosition): Iterator<MapTilePosition> {
|
||||
|
||||
// check if tile width/height should be divided by 2. I think I read that it's not divided by 2
|
||||
// if it's an odd number?
|
||||
|
||||
val xOffset: Int = floor(tileWidth.toDouble() / 2).toInt()
|
||||
val startX = origin.x - xOffset
|
||||
val endX = origin.x + xOffset
|
||||
|
||||
val yOffset: Int = floor(tileHeight.toDouble() / 2).toInt()
|
||||
val startY = origin.y - yOffset
|
||||
val endY = origin.y + yOffset
|
||||
|
||||
return iterator {
|
||||
for (x in (startX..endX)) {
|
||||
for (y in (startY..endY)) { // screen coordinates = +y points to the floor, -y to the ceiling
|
||||
yield(MapTilePosition(x, y))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun FactorioPrototype.Entity.mapColour(): ColourHex =
|
||||
(colour ?: mapColour ?: mapColourFriend)?.toHex() ?: ColourHex.WHITE
|
||||
|
||||
|
||||
private operator fun ColourHex.div(divisor: UInt): ColourHex = copy(
|
||||
red = (red / divisor).toUByte(),
|
||||
green = (green / divisor).toUByte(),
|
||||
blue = (blue / divisor).toUByte(),
|
||||
alpha = (alpha / divisor).toUByte(),
|
||||
)
|
||||
|
||||
|
||||
private operator fun ColourHex.plus(other: ColourHex): ColourHex = copy(
|
||||
red = (red + other.red).toUByte(),
|
||||
green = (green + other.green).toUByte(),
|
||||
blue = (blue + other.blue).toUByte(),
|
||||
alpha = (alpha + other.alpha).toUByte(),
|
||||
)
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.Tick
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_SRC_SERVER_LOG
|
||||
import dev.adamko.kafkatorio.processor.config.jsonMapper
|
||||
import dev.adamko.kafkatorio.processor.config.topicName
|
||||
import dev.adamko.kotka.extensions.component1
|
||||
import dev.adamko.kotka.extensions.component2
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.extensions.streams.to
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import java.util.zip.Inflater
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import okio.Buffer
|
||||
import okio.ByteString
|
||||
import okio.ByteString.Companion.decodeBase64
|
||||
import okio.buffer
|
||||
import okio.inflate
|
||||
import org.apache.kafka.common.serialization.Serdes
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
|
||||
|
||||
fun factorioServerPacketStream(
|
||||
builder: StreamsBuilder = StreamsBuilder(),
|
||||
): KStream<FactorioServerId, KafkatorioPacket> {
|
||||
|
||||
return builder.stream(
|
||||
TOPIC_SRC_SERVER_LOG,
|
||||
consumedAs(
|
||||
"read-raw-packets-from-server",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
Serdes.String(),
|
||||
)
|
||||
).mapValues("decode-packets") { _: FactorioServerId, value: String? ->
|
||||
|
||||
val packet = try {
|
||||
val decoded = decodeFactorioEncodedString(value!!)
|
||||
jsonMapper.decodeFromString(KafkatorioPacket.serializer(), decoded)
|
||||
} catch (e: Exception) {
|
||||
if (e is CancellationException) throw e
|
||||
|
||||
KafkatorioPacket(
|
||||
modVersion = "unknown",
|
||||
tick = Tick(0u),
|
||||
data = KafkatorioPacketDataError(
|
||||
exception = e,
|
||||
rawValue = value,
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
if (packet.data is KafkatorioPacketDataError) {
|
||||
println("error parsing $TOPIC_SRC_SERVER_LOG message: $packet")
|
||||
}
|
||||
|
||||
packet
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun splitFactorioServerPacketStream(
|
||||
factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>
|
||||
) {
|
||||
factorioServerPacketStream
|
||||
.to(
|
||||
producedAs(
|
||||
"split-server-log",
|
||||
jsonMapper.serde(),
|
||||
jsonMapper.serde(),
|
||||
)
|
||||
) { (_: FactorioServerId, value: KafkatorioPacket) ->
|
||||
// println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}")
|
||||
value.data.topicName
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Factorio `game.encode_string` zlib encodes a string, then base64's it. Here we do the reverse.
|
||||
*/
|
||||
private fun decodeFactorioEncodedString(
|
||||
source: String
|
||||
): String {
|
||||
val bytes: ByteString = source.decodeBase64() ?: error("failed to base64 decode $source")
|
||||
|
||||
return Buffer().write(bytes)
|
||||
.inflate(Inflater())
|
||||
.buffer()
|
||||
.readString(Charsets.UTF_8)
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import com.sksamuel.scrimage.color.RGBColor
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.MapTile
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import java.awt.Color
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
|
||||
@Serializable
|
||||
data class ServerMapChunkTiles<Data : Any>(
|
||||
val chunkId: ServerMapChunkId,
|
||||
val map: Map<MapTilePosition, Data>,
|
||||
) {
|
||||
operator fun plus(other: ServerMapChunkTiles<Data>) = copy(map = map + other.map)
|
||||
}
|
||||
|
||||
|
||||
val MapTile.position
|
||||
get() = MapTilePosition(x = x, y = y)
|
||||
|
||||
|
||||
val TRANSPARENT_AWT: Color = ColourHex.TRANSPARENT.toRgbColor().awt()
|
||||
|
||||
|
||||
fun ColourHex.toRgbColor(): RGBColor {
|
||||
return RGBColor(
|
||||
red.toInt(),
|
||||
green.toInt(),
|
||||
blue.toInt(),
|
||||
alpha.toInt(),
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.processor.config.jsonMapper
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
|
||||
import dev.adamko.kafkatorio.processor.config.topicName
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
|
||||
|
||||
inline fun <reified T : KafkatorioPacketData> StreamsBuilder.streamPacketData(): KStream<FactorioServerId, T> {
|
||||
|
||||
val simpleName = T::class.simpleName!!
|
||||
val topicName = T::class.topicName
|
||||
|
||||
val taskName = "consume.${simpleName}"
|
||||
|
||||
return stream<FactorioServerId, KafkatorioPacket>(
|
||||
topicName,
|
||||
consumedAs("$taskName.input", jsonMapper.serde(), jsonMapper.serde())
|
||||
).filter("$taskName.filter-is-instance") { key: FactorioServerId?, value: KafkatorioPacket? ->
|
||||
key != null && value?.data is T
|
||||
}.mapValues("$taskName.map-values") { _: FactorioServerId, packet: KafkatorioPacket ->
|
||||
packet.data as T
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioEntityData
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioPrototype
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeId
|
||||
import dev.adamko.kafkatorio.schema.common.toHex
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kotka.extensions.groupedAs
|
||||
import dev.adamko.kotka.extensions.materializedAs
|
||||
import dev.adamko.kotka.extensions.materializedWith
|
||||
import dev.adamko.kotka.extensions.*
|
||||
import dev.adamko.kotka.extensions.streams.*
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.extensions.streams.toTable
|
||||
import dev.adamko.kotka.extensions.tables.*
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlinx.serialization.Serializable
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
import org.apache.kafka.streams.kstream.KTable
|
||||
|
||||
|
||||
@JvmInline
|
||||
@Serializable
|
||||
value class TileColourDict(
|
||||
private val map: Map<PrototypeHashCode, ColourHex>
|
||||
) : Map<PrototypeHashCode, ColourHex> by map {
|
||||
operator fun plus(other: TileColourDict): TileColourDict = TileColourDict(this.map + other.map)
|
||||
}
|
||||
|
||||
|
||||
/** Get the latest map-tile colours per server. */
|
||||
fun KStream<FactorioServerId, PrototypesUpdate>.createTilePrototypeTable()
|
||||
: KTable<FactorioServerId, TileColourDict> {
|
||||
|
||||
val pid = "createTilePrototypeTable"
|
||||
|
||||
return mapValues(
|
||||
"$pid.map-values"
|
||||
) { _, protoPacket: PrototypesUpdate ->
|
||||
val map = protoPacket
|
||||
.prototypes
|
||||
.filterIsInstance<FactorioPrototype.MapTile>()
|
||||
.associate { PrototypeHashCode(it) to it.mapColour.toHex() }
|
||||
|
||||
TileColourDict(map)
|
||||
}.filter("$pid.filterMapTileProtos") { _, dict: TileColourDict ->
|
||||
dict.isNotEmpty()
|
||||
}.peek { serverId, dict: TileColourDict ->
|
||||
println("$pid: server $serverId has TileColourDict[${dict.size}]: ${dict.entries.joinToString()}")
|
||||
}.repartition(
|
||||
repartitionedAs(
|
||||
"$pid.pre-table-repartition",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
// force, otherwise KTable-KTable FK join doesn't work
|
||||
numberOfPartitions = 1,
|
||||
)
|
||||
).toTable(
|
||||
"$pid.create-table",
|
||||
materializedAs(
|
||||
"$pid.output-store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@JvmInline
|
||||
@Serializable
|
||||
value class EntityPrototypeMap(
|
||||
private val map: Map<PrototypeHashCode, FactorioPrototype.Entity>
|
||||
) : Map<PrototypeHashCode, FactorioPrototype.Entity> by map {
|
||||
operator fun plus(other: EntityPrototypeMap): EntityPrototypeMap =
|
||||
EntityPrototypeMap(this.map + other.map)
|
||||
|
||||
operator fun get(protoId: PrototypeId): FactorioPrototype.Entity? =
|
||||
map[PrototypeHashCode(protoId)]
|
||||
|
||||
operator fun get(entity: FactorioEntityData): FactorioPrototype.Entity? =
|
||||
map[PrototypeHashCode(entity.protoId)]
|
||||
}
|
||||
|
||||
|
||||
/** Get the latest entity colours per server. */
|
||||
fun KStream<FactorioServerId, PrototypesUpdate>.createEntityPrototypeMap(
|
||||
callerPid: String
|
||||
): KTable<FactorioServerId, EntityPrototypeMap> {
|
||||
|
||||
val pid = "$callerPid.createEntityPrototypeMap"
|
||||
|
||||
return mapValues("$pid.colour-dict.build") { _, protoPacket: PrototypesUpdate ->
|
||||
val map = protoPacket
|
||||
.prototypes
|
||||
.filterIsInstance<FactorioPrototype.Entity>()
|
||||
.associateBy { entityProto ->
|
||||
PrototypeHashCode(entityProto)
|
||||
}
|
||||
|
||||
EntityPrototypeMap(map)
|
||||
}.filter("$pid.filterEntityProtos") { _, dict: EntityPrototypeMap ->
|
||||
dict.isNotEmpty()
|
||||
}.groupByKey(groupedAs("$pid.group-by-key"))
|
||||
.reduce(
|
||||
"$pid.merge",
|
||||
materializedWith(
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
)
|
||||
) { dict1: EntityPrototypeMap, dict2: EntityPrototypeMap ->
|
||||
dict1 + dict2
|
||||
}.toStream("$pid.")
|
||||
.filter("$pid.not-null.filter") { _, v -> v != null }
|
||||
.mapValues("$pid.not-null.require") { _, value ->
|
||||
requireNotNull(value)
|
||||
}.peek { serverId, dict: EntityPrototypeMap ->
|
||||
println("$pid: server $serverId has EntityPrototypeMap (size:${dict.size}): ${dict.entries.joinToString()}")
|
||||
}.repartition(
|
||||
repartitionedAs(
|
||||
"$pid.pre-table-repartition",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
// force, otherwise KTable-KTable FK join doesn't work
|
||||
numberOfPartitions = 1,
|
||||
)
|
||||
).toTable(
|
||||
"$pid.create-table",
|
||||
materializedAs(
|
||||
"$pid.output-store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import com.sksamuel.scrimage.ImmutableImage
|
||||
import com.sksamuel.scrimage.ScaleMethod
|
||||
import com.sksamuel.scrimage.nio.PngWriter
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
|
||||
import dev.adamko.kafkatorio.schema.packets.EventServerPacket
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_BROADCAST_TO_WEBSOCKET
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
|
||||
import dev.adamko.kafkatorio.processor.config.jsonMapper
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.map
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import java.awt.image.BufferedImage
|
||||
import java.nio.file.Path
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.io.path.fileSize
|
||||
import kotlin.math.abs
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
|
||||
|
||||
private const val pid = "saveMapTiles"
|
||||
|
||||
private const val WEB_MAP_TILE_SIZE_PX = 256
|
||||
|
||||
|
||||
/**
|
||||
* @param[serverDataDir] [dev.adamko.kafkatorio.server.config.ApplicationProperties.serverDataDir]
|
||||
*/
|
||||
fun saveMapTiles(
|
||||
builder: StreamsBuilder,
|
||||
serverDataDir: Path,
|
||||
) {
|
||||
|
||||
val subdividedMapChunkTilesDebounced: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
builder.stream(
|
||||
consumedAs(
|
||||
"$pid.consume.grouped-map-chunks",
|
||||
kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
),
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
|
||||
TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
|
||||
TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
|
||||
).filter("$pid.consume.grouped-map-chunks.filter-size") { _, y ->
|
||||
y.chunkId.chunkSize.lengthInTiles >= ChunkSize.CHUNK_256.lengthInTiles
|
||||
}
|
||||
|
||||
val savedChunkFilenames: KStream<ServerMapChunkId, ServerMapTilePngFilename> =
|
||||
subdividedMapChunkTilesDebounced.saveChunkAsImage(serverDataDir)
|
||||
|
||||
savedChunkFilenames.broadcastSavedTileUpdates()
|
||||
}
|
||||
|
||||
|
||||
private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.saveChunkAsImage(
|
||||
tileDirectory: Path,
|
||||
): KStream<ServerMapChunkId, ServerMapTilePngFilename> = mapValues(
|
||||
"$pid.save-tiles"
|
||||
) { chunkId: ServerMapChunkId, tileColours: ServerMapChunkTiles<ColourHex> ->
|
||||
|
||||
val chunkImage = ImmutableImage.filled(
|
||||
chunkId.chunkSize.lengthInTiles,
|
||||
chunkId.chunkSize.lengthInTiles,
|
||||
TRANSPARENT_AWT,
|
||||
BufferedImage.TYPE_INT_ARGB
|
||||
)
|
||||
|
||||
val chunkOriginX: Int = chunkId.chunkPosition.x * chunkId.chunkSize.lengthInTiles
|
||||
val chunkOriginY: Int = chunkId.chunkPosition.y * chunkId.chunkSize.lengthInTiles
|
||||
|
||||
val validPixelRangeX = 0 until chunkImage.width
|
||||
val validPixelRangeY = 0 until chunkImage.height
|
||||
|
||||
tileColours.map.forEach { (tilePosition, colour) ->
|
||||
val rgbColour = colour.toRgbColor()
|
||||
|
||||
val pixelX = abs(abs(tilePosition.x) - abs(chunkOriginX))
|
||||
val pixelY = abs(abs(tilePosition.y) - abs(chunkOriginY))
|
||||
|
||||
if (!(pixelX in validPixelRangeX && pixelY in validPixelRangeY)) {
|
||||
// TODO figure out why buildings on chunk boundaries aren't re-chunked, and so cause this error
|
||||
println(
|
||||
"ERROR - pixel XY:[$pixelX,$pixelY] was out of bounds (x:$validPixelRangeX, y:$validPixelRangeY). $tilePosition, $colour, chunkOriginXY[$chunkOriginX,$chunkOriginY], $chunkId"
|
||||
)
|
||||
} else {
|
||||
try {
|
||||
chunkImage.setColor(pixelX, pixelY, rgbColour)
|
||||
} catch (e: Exception) {
|
||||
println(
|
||||
"ERROR $e - pixel XY:[$pixelX,$pixelY] (x:$validPixelRangeX, y:$validPixelRangeY). $tilePosition, $colour, chunkOriginXY[$chunkOriginX,$chunkOriginY], $chunkId"
|
||||
)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val scaledImage =
|
||||
chunkImage.scaleTo(WEB_MAP_TILE_SIZE_PX, WEB_MAP_TILE_SIZE_PX, ScaleMethod.FastScale)
|
||||
|
||||
val filename = ServerMapTilePngFilename(chunkId)
|
||||
saveTile(filename, tileDirectory, scaledImage)
|
||||
filename
|
||||
}
|
||||
|
||||
|
||||
@Synchronized
|
||||
private fun saveTile(
|
||||
filename: ServerMapTilePngFilename,
|
||||
serverDataDir: Path,
|
||||
img: ImmutableImage,
|
||||
) {
|
||||
val chunkImageFile = serverDataDir.resolve(filename.value).toFile()
|
||||
if (chunkImageFile.parentFile.mkdirs()) {
|
||||
println("created new map tile parentFile directory ${chunkImageFile.absolutePath}")
|
||||
}
|
||||
val sizeBefore = chunkImageFile.takeIf { it.exists() }?.toPath()?.fileSize()
|
||||
|
||||
val savedTile = img.output(PngWriter.NoCompression, chunkImageFile)
|
||||
|
||||
val sizeAfter = savedTile.takeIf { it.exists() }?.toPath()?.fileSize()
|
||||
|
||||
val sizeDescription = if (sizeBefore != sizeAfter && sizeBefore != null) {
|
||||
"$sizeBefore/$sizeAfter"
|
||||
} else {
|
||||
"$sizeAfter"
|
||||
}
|
||||
println("savedTile $sizeDescription: $savedTile")
|
||||
}
|
||||
|
||||
|
||||
private fun KStream<ServerMapChunkId, ServerMapTilePngFilename>.broadcastSavedTileUpdates() {
|
||||
map<ServerMapChunkId, ServerMapTilePngFilename, FactorioServerId, EventServerPacket>(
|
||||
"$pid.broadcast.filename-update.convert",
|
||||
) { id, filename ->
|
||||
id.serverId to EventServerPacket.ChunkTileSaved(id, filename)
|
||||
}.to(
|
||||
TOPIC_BROADCAST_TO_WEBSOCKET,
|
||||
producedAs(
|
||||
"$pid.broadcast.filename-update.send",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(EventServerPacket.serializer()),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.packets.EventServerPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.SurfaceUpdate
|
||||
import dev.adamko.kafkatorio.processor.config.TOPIC_BROADCAST_TO_WEBSOCKET
|
||||
import dev.adamko.kafkatorio.processor.config.topicName
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.forEach
|
||||
import dev.adamko.kotka.extensions.streams.map
|
||||
import dev.adamko.kotka.extensions.streams.to
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.serialization.encodeToString
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
|
||||
|
||||
private val jsonMapper: Json = Json {
|
||||
prettyPrint = false
|
||||
serializersModule = jsonMapperKafkatorio.serializersModule
|
||||
}
|
||||
|
||||
|
||||
private const val pid = "websocketBroadcast"
|
||||
|
||||
|
||||
interface WebsocketBroadcaster {
|
||||
suspend fun sendMessageToClients(message: String)
|
||||
}
|
||||
|
||||
|
||||
fun broadcastToWebsocket(
|
||||
websocketBroadcaster: WebsocketBroadcaster,
|
||||
builder: StreamsBuilder,
|
||||
) {
|
||||
|
||||
builder.broadcastKafkatorioPackets()
|
||||
|
||||
builder.stream<FactorioServerId, EventServerPacket>(
|
||||
TOPIC_BROADCAST_TO_WEBSOCKET,
|
||||
consumedAs("$pid.packets-to-broadcast.consume", jsonMapper.serde(), jsonMapper.serde()),
|
||||
).forEach(
|
||||
"$pid.websocket-server.send-message"
|
||||
) { _, esPacket ->
|
||||
runBlocking {
|
||||
websocketBroadcaster.sendMessageToClients(jsonMapper.encodeToString(esPacket))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun StreamsBuilder.broadcastKafkatorioPackets() {
|
||||
val playerUpdatesStream = stream(
|
||||
consumedAs(
|
||||
"$pid.player-updates",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(KafkatorioPacket.serializer()),
|
||||
),
|
||||
PlayerUpdate::class.topicName,
|
||||
SurfaceUpdate::class.topicName,
|
||||
)
|
||||
|
||||
playerUpdatesStream.map<FactorioServerId, KafkatorioPacket, FactorioServerId, EventServerPacket>(
|
||||
"$pid.packet-updates.convert"
|
||||
) { serverId, packet ->
|
||||
serverId to EventServerPacket.Kafkatorio(packet)
|
||||
}.to(
|
||||
producedAs(
|
||||
"$pid.event-server-packets.produce",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(EventServerPacket.serializer()),
|
||||
)
|
||||
) { TOPIC_BROADCAST_TO_WEBSOCKET }
|
||||
}
|
||||
|
||||
|
||||
private fun StreamsBuilder.prepareServerUpdateBroadcast() {
|
||||
|
||||
val surfaceUpdatesStream = stream<FactorioServerId, KafkatorioPacket>(
|
||||
SurfaceUpdate::class.topicName,
|
||||
consumedAs("$pid.surface-updates", jsonMapper.serde(), jsonMapper.serde()),
|
||||
)
|
||||
|
||||
surfaceUpdatesStream.map<FactorioServerId, KafkatorioPacket, FactorioServerId, EventServerPacket>(
|
||||
"$pid.surface-updates.convert"
|
||||
) { serverId, serverUpdatePacket ->
|
||||
serverId to EventServerPacket.Kafkatorio(serverUpdatePacket)
|
||||
}.to(
|
||||
producedAs(
|
||||
"$pid.player-updates.stream",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(EventServerPacket.serializer()),
|
||||
)
|
||||
) { TOPIC_BROADCAST_TO_WEBSOCKET }
|
||||
}
|
||||
67
modules/events-processor-core/src/main/resources/config.yml
Normal file
67
modules/events-processor-core/src/main/resources/config.yml
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
# Factorio Events Processor config
|
||||
|
||||
web-port: ${SERVER_PORT:-8080}
|
||||
server-data-dir: ${SERVER_DATA_DIR:-./server-data}
|
||||
|
||||
|
||||
socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}
|
||||
socket-server-port: ${SOCKET_SERVER_PORT:-12092}
|
||||
|
||||
|
||||
kafka-admin-config:
|
||||
client.id: kafkatorio.events-server-admin
|
||||
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}
|
||||
|
||||
|
||||
kafka-streams-config:
|
||||
application.id: kafkatorio-events-processor
|
||||
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}
|
||||
state.dir: ${KAFKA_STREAMS_STATE_DIR:-.state/kafka}
|
||||
num.stream.threads: 1
|
||||
processing.guarantee: exactly_once_v2
|
||||
auto.offset.reset: earliest
|
||||
default.key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
default.value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
|
||||
# default.deserialization.exception.handler: org.apache.kafka.streams.errors.LogAndContinueExceptionHandler
|
||||
|
||||
#buffered.records.per.partition: 0
|
||||
#poll.ms: 10000
|
||||
|
||||
# 31457280 : 30MB
|
||||
max.request.size: 31457280
|
||||
buffer.memory: 31457280
|
||||
|
||||
compression.type: snappy
|
||||
# linger.ms=5
|
||||
|
||||
# fetch.message.max.bytes=31457280
|
||||
# 100mb 104857600
|
||||
|
||||
|
||||
#kafka:
|
||||
# bootstrap-servers:
|
||||
# - "http://localhost:9092"
|
||||
# config: |
|
||||
# application.id=kafkatorio-events-processor
|
||||
# bootstrap.servers={{bootstrap-servers}}
|
||||
# state.dir=./kafka-state
|
||||
# num.stream.threads=1
|
||||
# processing.guarantee=exactly_once_v2
|
||||
# auto.offset.reset=earliest
|
||||
# default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
# default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
# default.deserialization.exception.handler=org.apache.kafka.streams.errors.LogAndContinueExceptionHandler
|
||||
# #buffered.records.per.partition=0
|
||||
# #poll.ms=10000
|
||||
#
|
||||
# # 31457280 = 30MB
|
||||
# max.request.size=31457280
|
||||
# buffer.memory=31457280
|
||||
#
|
||||
# compression.type=snappy
|
||||
# # linger.ms=5
|
||||
#
|
||||
# # fetch.message.max.bytes=31457280
|
||||
# # 100mb 104857600
|
||||
#
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
<root>
|
||||
<item name='org.apache.kafka.streams.state.ReadOnlyKeyValueStore V get(K)'>
|
||||
<annotation name='org.jetbrains.annotations.Nullable'/>
|
||||
</item>
|
||||
</root>
|
||||
21
modules/events-processor-core/src/main/resources/logback.xml
Normal file
21
modules/events-processor-core/src/main/resources/logback.xml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
<configuration>
|
||||
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<withJansi>false</withJansi>
|
||||
<!-- encoders are assigned the type
|
||||
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
|
||||
<encoder>
|
||||
<pattern>%cyan(%d{HH:mm:ss.SSS}) %white([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n
|
||||
</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="org.apache.kafka" level="info" additivity="false">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</logger>
|
||||
|
||||
<root level="info">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
package dev.adamko.kafkatorio.processor.core
|
||||
|
||||
import dev.adamko.kafkatorio.schema.common.MaskedValue
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.config.PortVal
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import io.kotest.matchers.nulls.shouldBeNull
|
||||
import io.kotest.matchers.nulls.shouldNotBeNull
|
||||
import kotlin.io.path.Path
|
||||
|
||||
|
||||
class AuthenticatorTest : FunSpec({
|
||||
|
||||
test("verify correct JWT is verified") {
|
||||
|
||||
val authenticator = Authenticator(
|
||||
testApplicationProperties(
|
||||
jwtSecret = "123 456 789"
|
||||
)
|
||||
)
|
||||
|
||||
val result = authenticator.verifyJwt(jwtMessage)
|
||||
|
||||
result.shouldNotBeNull()
|
||||
}
|
||||
|
||||
test("verify invalid JWT returns null") {
|
||||
|
||||
val authenticator = Authenticator(
|
||||
testApplicationProperties(
|
||||
jwtSecret = "wrong"
|
||||
)
|
||||
)
|
||||
|
||||
val result = authenticator.verifyJwt(jwtMessage)
|
||||
|
||||
result.shouldBeNull()
|
||||
}
|
||||
|
||||
}) {
|
||||
|
||||
companion object {
|
||||
/** Key: `123 465 789` */
|
||||
private val jwtMessage = """
|
||||
eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.ToFLkPBs_EKF4B2bRQyVPfkKrB-x6WgyAq6KsR9oHwg
|
||||
""".trimIndent()
|
||||
|
||||
private fun testApplicationProperties(
|
||||
jwtSecret: String,
|
||||
) = ApplicationProperties(
|
||||
webPort = PortVal.DEFAULT,
|
||||
serverDataDir = Path(""),
|
||||
kafkaAdminConfig = emptyMap(),
|
||||
kafkaStreamsConfig = emptyMap(),
|
||||
socketServerHost = "",
|
||||
socketServerPort = 9999,
|
||||
jwtSecret = MaskedValue(jwtSecret),
|
||||
kafkatorioServers = emptyMap(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
package dev.adamko.kafkatorio.processor.misc
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
|
||||
import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
|
||||
import java.util.UUID
|
||||
import kotlinx.serialization.encodeToByteArray
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
|
||||
class Blah {
|
||||
|
||||
@Test
|
||||
fun asdad() {
|
||||
|
||||
val tiles = ServerMapChunkTiles<ColourHex>(
|
||||
chunkId = ServerMapChunkId(
|
||||
serverId = FactorioServerId(UUID.randomUUID().toString()),
|
||||
layer = ServerMapTileLayer.BUILDING,
|
||||
chunkPosition = MapChunkPosition(Int.MAX_VALUE, Int.MAX_VALUE, ChunkSize.MAX),
|
||||
surfaceIndex = SurfaceIndex(UInt.MAX_VALUE),
|
||||
),
|
||||
buildMap {
|
||||
repeat(512) { x ->
|
||||
repeat(512) { y ->
|
||||
put(MapTilePosition(x, y), ColourHex(255u, 255u, 255u, 255u))
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
val encode = kxsBinary.encodeToByteArray(tiles)
|
||||
|
||||
encode.inputStream()
|
||||
val size = encode.size * Byte.SIZE_BYTES
|
||||
|
||||
println(size)
|
||||
|
||||
|
||||
val map2 = buildList {
|
||||
repeat(512) { x ->
|
||||
repeat(512) { y ->
|
||||
val element = (x to y) to Int.MAX_VALUE
|
||||
add(element)
|
||||
}
|
||||
}
|
||||
}
|
||||
// .toTypedArray()
|
||||
val encode2 = kxsBinary.encodeToByteArray(map2)
|
||||
|
||||
encode2.inputStream()
|
||||
val size2 = encode2.size * Byte.SIZE_BYTES
|
||||
|
||||
println(size2)
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
package dev.adamko.kafkatorio.processor.core.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.toJavaDuration
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.TestInputTopic
|
||||
import org.apache.kafka.streams.TopologyTestDriver
|
||||
|
||||
class FactorioServerPacketsTest : FunSpec({
|
||||
|
||||
|
||||
})
|
||||
|
||||
private class Scenario(
|
||||
streamsBuilder: StreamsBuilder = StreamsBuilder(),
|
||||
|
||||
groupedMapChunksInputTopicName: String = "kafkatorio.state.map-chunk.terrain.colour",
|
||||
|
||||
val serverId: FactorioServerId = FactorioServerId("test-server-id"),
|
||||
) : AutoCloseable {
|
||||
|
||||
init {
|
||||
factorioServerPacketStream(streamsBuilder)
|
||||
}
|
||||
|
||||
private val testDriver: TopologyTestDriver = TopologyTestDriver(streamsBuilder.build())
|
||||
|
||||
val groupedMapChunksInputTopic: TestInputTopic<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
testDriver.createInputTopic(
|
||||
groupedMapChunksInputTopicName,
|
||||
kxsBinary.serde<ServerMapChunkId>().serializer(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>().serializer(),
|
||||
)
|
||||
|
||||
operator fun plusAssign(wallClockTime: Duration) {
|
||||
testDriver.advanceWallClockTime(wallClockTime.toJavaDuration())
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
testDriver.close()
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,185 @@
|
|||
package dev.adamko.kafkatorio.processor.core.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.iterateTiles
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.library.toMapChunkPosition
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import io.kotest.property.Arb
|
||||
import io.kotest.property.PropTestConfig
|
||||
import io.kotest.property.RandomSource
|
||||
import io.kotest.property.arbitrary.arbitrary
|
||||
import io.kotest.property.arbitrary.int
|
||||
import io.kotest.property.arbitrary.uByte
|
||||
import io.kotest.property.checkAll
|
||||
import java.nio.file.Path
|
||||
import java.time.LocalTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import kotlin.io.path.Path
|
||||
import kotlin.io.path.absolutePathString
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.minutes
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import kotlin.time.toJavaDuration
|
||||
import kotlinx.coroutines.delay
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.TestInputTopic
|
||||
import org.apache.kafka.streams.TopologyTestDriver
|
||||
|
||||
|
||||
class SaveMapChunksTest : FunSpec({
|
||||
|
||||
test("save tile") {
|
||||
|
||||
Scenario().use { scenario ->
|
||||
|
||||
val chunkTilesArb = serverMapChunkTilesArb(scenario.serverId)
|
||||
|
||||
checkAll(
|
||||
config = PropTestConfig(iterations = 1),
|
||||
chunkTilesArb,
|
||||
) { chunkTiles ->
|
||||
|
||||
scenario.groupedMapChunksInputTopic.pipeInput(
|
||||
chunkTiles.chunkId,
|
||||
chunkTiles,
|
||||
)
|
||||
|
||||
scenario += 1.minutes
|
||||
delay(5.seconds)
|
||||
scenario += 1.minutes
|
||||
// delay(5.seconds)
|
||||
scenario += 1.minutes
|
||||
// delay(5.seconds)
|
||||
scenario += 1.minutes
|
||||
// delay(5.seconds)
|
||||
|
||||
println("file:///${scenario.outputDir.absolutePathString()}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}) {
|
||||
|
||||
class Scenario(
|
||||
streamsBuilder: StreamsBuilder = StreamsBuilder(),
|
||||
|
||||
groupedMapChunksInputTopicName: String = "kafkatorio.state.map-chunk.terrain.colour",
|
||||
|
||||
val serverId: FactorioServerId = FactorioServerId("test-server-id"),
|
||||
) : AutoCloseable {
|
||||
|
||||
val outputDir: Path = run {
|
||||
val now = LocalTime.now().format(DateTimeFormatter.ISO_TIME).filter { it.isLetterOrDigit() }
|
||||
Path("build/test/save-map-chunks-${now}")
|
||||
}
|
||||
private val topology = saveMapTiles(streamsBuilder, outputDir)
|
||||
private val testDriver: TopologyTestDriver = TopologyTestDriver(topology)
|
||||
|
||||
val groupedMapChunksInputTopic: TestInputTopic<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
testDriver.createInputTopic(
|
||||
groupedMapChunksInputTopicName,
|
||||
kxsBinary.serde<ServerMapChunkId>().serializer(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>().serializer(),
|
||||
)
|
||||
|
||||
operator fun plusAssign(wallClockTime: Duration) {
|
||||
testDriver.advanceWallClockTime(wallClockTime.toJavaDuration())
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
testDriver.close()
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
|
||||
private fun mapChunkPositionArb(
|
||||
xArb: Arb<Int> = Arb.int(-10..10),
|
||||
yArb: Arb<Int> = Arb.int(-10..10),
|
||||
chunkSize: ChunkSize = ChunkSize.MAX,
|
||||
): Arb<MapChunkPosition> = arbitrary {
|
||||
MapChunkPosition(xArb.bind(), yArb.bind(), chunkSize)
|
||||
}
|
||||
|
||||
private fun colourHexArb(
|
||||
redArb: Arb<UByte> = Arb.uByte(),
|
||||
greenArb: Arb<UByte> = Arb.uByte(),
|
||||
blueArb: Arb<UByte> = Arb.uByte(),
|
||||
alphaArb: Arb<UByte> = Arb.uByte(),
|
||||
): Arb<ColourHex> = arbitrary {
|
||||
ColourHex(
|
||||
redArb.bind(),
|
||||
greenArb.bind(),
|
||||
blueArb.bind(),
|
||||
alphaArb.bind(),
|
||||
)
|
||||
}
|
||||
|
||||
private fun serverMapChunkIdArb(
|
||||
serverId: FactorioServerId,
|
||||
chunkSize: ChunkSize = ChunkSize.MAX,
|
||||
chunkPosition: Arb<MapChunkPosition> = mapChunkPositionArb(chunkSize = chunkSize),
|
||||
surfaceIndex: SurfaceIndex = SurfaceIndex(1u),
|
||||
): Arb<ServerMapChunkId> = arbitrary {
|
||||
ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.RESOURCE,
|
||||
chunkPosition = chunkPosition.bind(),
|
||||
surfaceIndex = surfaceIndex,
|
||||
)
|
||||
}
|
||||
|
||||
private fun tilesInChunkArb(
|
||||
chunkId: ServerMapChunkId
|
||||
): Arb<Map<MapTilePosition, ColourHex>> = arbitrary {
|
||||
|
||||
val tileColours = mutableMapOf<MapChunkPosition, ColourHex>()
|
||||
|
||||
println("chunk size: ${chunkId.chunkSize} ${chunkId.chunkSize.lengthInTiles}")
|
||||
|
||||
val map = chunkId.chunkPosition
|
||||
.iterateTiles()
|
||||
.asSequence()
|
||||
.map { tilePos ->
|
||||
val colour =
|
||||
tileColours.getOrPut(tilePos.toMapChunkPosition(ChunkSize.MIN)) {
|
||||
colourHexArb().sample(RandomSource.default()).value
|
||||
}
|
||||
tilePos to colour
|
||||
}
|
||||
// .take(100)
|
||||
.toMap()
|
||||
|
||||
println(map.size)
|
||||
|
||||
map
|
||||
}
|
||||
|
||||
fun serverMapChunkTilesArb(
|
||||
serverId: FactorioServerId,
|
||||
): Arb<ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val serverMapChunkId = serverMapChunkIdArb(serverId)
|
||||
|
||||
return arbitrary {
|
||||
val chunkId = serverMapChunkId.bind()
|
||||
|
||||
val tileColoursArb = tilesInChunkArb(chunkId)
|
||||
|
||||
ServerMapChunkTiles(
|
||||
chunkId = chunkId,
|
||||
map = tileColoursArb.bind()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue