big ol' refactor to split up the events-server and events-processors
This commit is contained in:
Adam
parent
d82f25c6ee
commit
873983f9a3
71 changed files with 1013 additions and 757 deletions
|
|
@ -10,11 +10,11 @@ plugins {
|
|||
}
|
||||
|
||||
|
||||
description = "serves Kafkatorio content over the web"
|
||||
description = "deprecated - this was split up into separate modules"
|
||||
|
||||
val projectId: String by project.extra
|
||||
|
||||
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
|
||||
val kafkaStateDir: Directory = layout.projectDirectory.dir(".state/kafka")
|
||||
|
||||
|
||||
dependencies {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ ENV SOCKET_SERVER_PORT=9002
|
|||
EXPOSE $SOCKET_SERVER_PORT
|
||||
|
||||
ENV SERVER_DATA_DIR="/app/data"
|
||||
ENV KAFKA_STREAMS_STATE_DIR="/app/kafka-state"
|
||||
ENV KAFKA_STREAMS_STATE_DIR="/app/state/kafka"
|
||||
VOLUME ${SERVER_DATA_DIR} \
|
||||
${KAFKA_STREAMS_STATE_DIR}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.config
|
||||
|
||||
import com.sksamuel.hoplite.ConfigLoaderBuilder
|
||||
import com.sksamuel.hoplite.addEnvironmentSource
|
||||
import com.sksamuel.hoplite.addResourceOrFileSource
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerToken
|
||||
import dev.adamko.kafkatorio.schema.common.MaskedValue
|
||||
import java.nio.file.Path
|
||||
|
||||
|
||||
// loaded by hoplite
|
||||
data class ApplicationProperties(
|
||||
val webPort: PortVal,
|
||||
val serverDataDir: Path,
|
||||
val kafkaAdminConfig: Map<String, String>,
|
||||
val kafkaStreamsConfig: Map<String, String>,
|
||||
|
||||
val socketServerHost: String,
|
||||
val socketServerPort: Int,
|
||||
|
||||
val jwtSecret: MaskedValue,
|
||||
|
||||
val kafkatorioServers: Map<FactorioServerToken, FactorioServerId>
|
||||
) {
|
||||
|
||||
companion object {
|
||||
/** use Hoplite to load config from environment variables or properties files */
|
||||
fun load(): ApplicationProperties =
|
||||
ConfigLoaderBuilder.default()
|
||||
.addDefaults()
|
||||
.allowUnresolvedSubstitutions()
|
||||
.allowEmptySources()
|
||||
.addEnvironmentSource()
|
||||
.addResourceOrFileSource("/.secret.config.yml", optional = true)
|
||||
.addResourceOrFileSource("/config.yml", optional = false)
|
||||
.build()
|
||||
.loadConfigOrThrow()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.config
|
||||
|
||||
|
||||
/** Improved (slightly) version of http4k's `Port`. */
|
||||
@JvmInline
|
||||
value class PortVal(val value: Int) {
|
||||
|
||||
init {
|
||||
require(value in VALID_RANGE) {
|
||||
"Out of range. Port: '$value', range: $VALID_RANGE"
|
||||
}
|
||||
}
|
||||
|
||||
operator fun invoke(): Int = value
|
||||
|
||||
companion object {
|
||||
val DEFAULT = PortVal(8080)
|
||||
val VALID_RANGE: IntRange
|
||||
get() = (1..65535) // get() is required to prevent init ordering causing NPE
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.config
|
||||
|
||||
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
|
||||
import kotlinx.serialization.json.Json
|
||||
|
||||
|
||||
val jsonMapper: Json = Json {
|
||||
prettyPrint = true
|
||||
prettyPrintIndent = " "
|
||||
serializersModule = jsonMapperKafkatorio.serializersModule
|
||||
}
|
||||
|
|
@ -1,32 +1,32 @@
|
|||
package dev.adamko.kafkatorio.server
|
||||
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.processor.eventsProcessor
|
||||
import dev.adamko.kafkatorio.server.security.Authenticator
|
||||
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
|
||||
import dev.adamko.kafkatorio.server.web.startWebServer
|
||||
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
//import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
//import dev.adamko.kafkatorio.server.processor.eventsProcessor
|
||||
//import dev.adamko.kafkatorio.server.security.Authenticator
|
||||
//import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
|
||||
//import dev.adamko.kafkatorio.server.web.startWebServer
|
||||
//import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
//import kotlinx.coroutines.coroutineScope
|
||||
//import kotlinx.coroutines.launch
|
||||
|
||||
|
||||
suspend fun main(): Unit = coroutineScope {
|
||||
|
||||
val appProps = ApplicationProperties.load()
|
||||
|
||||
val websocketServer = WebmapWebsocketServer()
|
||||
val syslogSocketServer = SyslogSocketServer(appProps)
|
||||
val authenticator = Authenticator(appProps)
|
||||
|
||||
launch {
|
||||
startWebServer(appProps, websocketServer)
|
||||
}
|
||||
|
||||
launch {
|
||||
syslogSocketServer.start()
|
||||
}
|
||||
|
||||
launch {
|
||||
eventsProcessor(appProps, syslogSocketServer, websocketServer, authenticator)
|
||||
}
|
||||
}
|
||||
//suspend fun main(): Unit = coroutineScope {
|
||||
//
|
||||
// val appProps = ApplicationProperties.load()
|
||||
//
|
||||
// val websocketServer = WebmapWebsocketServer()
|
||||
// val syslogSocketServer = SyslogSocketServer(appProps)
|
||||
// val authenticator = Authenticator(appProps)
|
||||
//
|
||||
// launch {
|
||||
// startWebServer(appProps, websocketServer)
|
||||
// }
|
||||
//
|
||||
// launch {
|
||||
// syslogSocketServer.start()
|
||||
// }
|
||||
//
|
||||
// launch {
|
||||
// eventsProcessor(appProps, syslogSocketServer, websocketServer, authenticator)
|
||||
// }
|
||||
//}
|
||||
|
|
|
|||
|
|
@ -1,149 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor
|
||||
|
||||
import dev.adamko.kotka.extensions.component1
|
||||
import dev.adamko.kotka.extensions.component2
|
||||
import dev.adamko.kotka.extensions.component3
|
||||
import dev.adamko.kotka.extensions.processor.component1
|
||||
import dev.adamko.kotka.extensions.processor.component2
|
||||
import dev.adamko.kotka.extensions.processor.component3
|
||||
import dev.adamko.kotka.extensions.state.useAll
|
||||
import kotlin.time.toJavaDuration
|
||||
import org.apache.kafka.common.serialization.Serde
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.processor.Cancellable
|
||||
import org.apache.kafka.streams.processor.PunctuationType
|
||||
import org.apache.kafka.streams.processor.api.Processor
|
||||
import org.apache.kafka.streams.processor.api.ProcessorContext
|
||||
import org.apache.kafka.streams.processor.api.ProcessorSupplier
|
||||
import org.apache.kafka.streams.processor.api.Record
|
||||
import org.apache.kafka.streams.state.StoreBuilder
|
||||
import org.apache.kafka.streams.state.Stores
|
||||
import org.apache.kafka.streams.state.TimestampedKeyValueStore
|
||||
import org.apache.kafka.streams.state.ValueAndTimestamp
|
||||
import kotlin.time.Duration
|
||||
|
||||
|
||||
class DebounceProcessor<K, V>(
|
||||
private val inactivityDuration: Duration,
|
||||
private val checkInterval: Duration,
|
||||
private val storeName: String,
|
||||
) : Processor<K, V, K, V> {
|
||||
|
||||
private lateinit var state: State<K, V>
|
||||
|
||||
|
||||
override fun init(context: ProcessorContext<K, V>) {
|
||||
state = Active(context)
|
||||
}
|
||||
|
||||
|
||||
override fun process(record: Record<K, V>) {
|
||||
(state as Active).apply {
|
||||
val (key, value, _) = record
|
||||
|
||||
val existing: ValueAndTimestamp<V>? = store.get(key)
|
||||
val newTimestamp = existing?.timestamp() ?: System.currentTimeMillis()
|
||||
|
||||
store.put(key, ValueAndTimestamp.make(value, newTimestamp))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
override fun close() {
|
||||
(state as? Active)?.schedule?.cancel()
|
||||
state = Closed()
|
||||
}
|
||||
|
||||
|
||||
private sealed interface State<K, V>
|
||||
|
||||
|
||||
private class Closed<K, V> : State<K, V>
|
||||
|
||||
|
||||
private inner class Active(
|
||||
val context: ProcessorContext<K, V>
|
||||
) : State<K, V> {
|
||||
val store: TimestampedKeyValueStore<K, V> = context.getStateStore(storeName)
|
||||
val schedule: Cancellable = context.schedule(
|
||||
checkInterval.toJavaDuration(),
|
||||
PunctuationType.WALL_CLOCK_TIME,
|
||||
::checkRecords
|
||||
)
|
||||
|
||||
private fun checkRecords(now: Long) {
|
||||
store.useAll {
|
||||
forEach { record ->
|
||||
val (key, value, timestamp) = record
|
||||
if ((now - timestamp) > inactivityDuration.inWholeMilliseconds) {
|
||||
context.forward(Record(key, value, timestamp))
|
||||
store.delete(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
|
||||
fun <K, V> Topology.addDebounceProcessor(
|
||||
namePrefix: String,
|
||||
sourceTopic: String,
|
||||
sinkTopic: String,
|
||||
|
||||
inactivityDuration: Duration,
|
||||
checkInterval: Duration = inactivityDuration / 10,
|
||||
keySerde: Serde<K>,
|
||||
valueSerde: Serde<V>,
|
||||
storeName: String = "$namePrefix.debounce-store",
|
||||
): Topology {
|
||||
|
||||
val sourceName = "$namePrefix.debounce-source"
|
||||
val processorName = "$namePrefix.debounce-processor"
|
||||
val sinkName = "$namePrefix.debounce-sink"
|
||||
|
||||
return addSource(
|
||||
sourceName,
|
||||
keySerde.deserializer(),
|
||||
valueSerde.deserializer(),
|
||||
sourceTopic,
|
||||
).addProcessor(
|
||||
processorName,
|
||||
Supplier(
|
||||
inactivityDuration = inactivityDuration,
|
||||
checkInterval = checkInterval,
|
||||
storeName = storeName,
|
||||
keySerde = keySerde,
|
||||
valueSerde = valueSerde,
|
||||
),
|
||||
sourceName,
|
||||
).addSink(
|
||||
sinkName,
|
||||
sinkTopic,
|
||||
keySerde.serializer(),
|
||||
valueSerde.serializer(),
|
||||
processorName,
|
||||
)
|
||||
}
|
||||
|
||||
class Supplier<K, V>(
|
||||
private val inactivityDuration: Duration,
|
||||
private val checkInterval: Duration,
|
||||
private val storeName: String,
|
||||
private val keySerde: Serde<K>,
|
||||
private val valueSerde: Serde<V>,
|
||||
) : ProcessorSupplier<K, V, K, V> {
|
||||
|
||||
override fun get(): Processor<K, V, K, V> =
|
||||
DebounceProcessor(inactivityDuration, checkInterval, storeName)
|
||||
|
||||
override fun stores(): Set<StoreBuilder<*>> = setOf(
|
||||
Stores.timestampedKeyValueStoreBuilder(
|
||||
Stores.persistentTimestampedKeyValueStore(storeName),
|
||||
keySerde,
|
||||
valueSerde,
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor
|
||||
|
||||
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.config.jsonMapper
|
||||
import dev.adamko.kafkatorio.server.security.Authenticator
|
||||
import dev.adamko.kafkatorio.server.socket.SyslogMsg
|
||||
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
|
||||
import dev.adamko.kotka.kxs.kafkaSerializer
|
||||
import java.util.Properties
|
||||
import kotlin.coroutines.coroutineContext
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.flow.collect
|
||||
import kotlinx.coroutines.flow.filter
|
||||
import kotlinx.coroutines.flow.onEach
|
||||
import kotlinx.coroutines.job
|
||||
import org.apache.kafka.clients.producer.KafkaProducer
|
||||
import org.apache.kafka.clients.producer.ProducerRecord
|
||||
import org.apache.kafka.common.header.internals.RecordHeader
|
||||
import org.apache.kafka.common.serialization.Serdes
|
||||
import org.apache.kafka.streams.StreamsConfig
|
||||
|
||||
|
||||
/**
|
||||
* Consume Syslog messages, verifies each, and forwards them to a Kafka topic.
|
||||
*/
|
||||
class KafkatorioPacketKafkaProducer(
|
||||
private val appProps: ApplicationProperties,
|
||||
private val syslogServer: SyslogSocketServer,
|
||||
private val authenticator: Authenticator,
|
||||
) : AutoCloseable {
|
||||
|
||||
private val props: Properties = appProps.kafkaStreamsConfig.toProperties()
|
||||
|
||||
init {
|
||||
props.updateValue(StreamsConfig.APPLICATION_ID_CONFIG) { "$it.KafkatorioPacketKafkaProducer" }
|
||||
}
|
||||
|
||||
private val producer: KafkaProducer<FactorioServerId, String> = KafkaProducer(
|
||||
props.apply {
|
||||
updateValue(StreamsConfig.APPLICATION_ID_CONFIG) { "$it.KafkatorioPacketKafkaProducer" }
|
||||
},
|
||||
jsonMapper.kafkaSerializer(FactorioServerId.serializer()),
|
||||
Serdes.String().serializer(),
|
||||
)
|
||||
|
||||
private val dlqProducer: KafkaProducer<String, String> = KafkaProducer(
|
||||
props.apply {
|
||||
updateValue(StreamsConfig.APPLICATION_ID_CONFIG) { "$it.KafkatorioPacketKafkaProducer.dlq" }
|
||||
},
|
||||
Serdes.String().serializer(),
|
||||
Serdes.String().serializer(),
|
||||
)
|
||||
|
||||
|
||||
suspend fun launch() {
|
||||
coroutineContext.job.invokeOnCompletion {
|
||||
this@KafkatorioPacketKafkaProducer.close()
|
||||
}
|
||||
|
||||
syslogServer.messages
|
||||
.filter {
|
||||
(it.message ?: "").run {
|
||||
startsWith("KafkatorioPacket:::") || startsWith("KafkatorioPacket encoded:::")
|
||||
}
|
||||
}.onEach { syslog ->
|
||||
runCatching {
|
||||
getFactorioServerId(syslog)
|
||||
}.fold(
|
||||
onSuccess = { serverId -> produceMessage(serverId, syslog) },
|
||||
onFailure = { ex ->
|
||||
produceDlqMessage(syslog, "${ex::class.java.name}: ${ex.message}")
|
||||
|
||||
if (ex is CancellationException) throw ex
|
||||
}
|
||||
)
|
||||
}.collect()
|
||||
}
|
||||
|
||||
|
||||
private fun getFactorioServerId(syslog: SyslogMsg): FactorioServerId {
|
||||
val token = syslog.headerAppName
|
||||
if (token.isNullOrBlank()) {
|
||||
error("no token in syslog message. msg: ${syslog.src}")
|
||||
}
|
||||
|
||||
val serverToken = authenticator.verifyJwt(token)
|
||||
if (serverToken.isNullOrBlank()) {
|
||||
error("could not decode token $token. msg: ${syslog.src}")
|
||||
}
|
||||
|
||||
val serverId = appProps.kafkatorioServers[serverToken]
|
||||
if (serverId.isNullOrBlank()) {
|
||||
error("no server ID for token $serverToken. msg: ${syslog.src}")
|
||||
}
|
||||
|
||||
return serverId
|
||||
}
|
||||
|
||||
|
||||
private fun produceMessage(
|
||||
serverId: FactorioServerId,
|
||||
syslog: SyslogMsg,
|
||||
) {
|
||||
val kafkatorioMessage = syslog.message?.substringAfter(":::")
|
||||
|
||||
val record = ProducerRecord<FactorioServerId, String>(
|
||||
TOPIC_SRC_SERVER_LOG,
|
||||
serverId,
|
||||
kafkatorioMessage,
|
||||
)
|
||||
|
||||
syslog.copyHeadersTo(record)
|
||||
|
||||
producer.send(record)
|
||||
}
|
||||
|
||||
|
||||
private fun produceDlqMessage(syslog: SyslogMsg, errorMessage: String) {
|
||||
log("error: $errorMessage, $syslog")
|
||||
|
||||
val record = ProducerRecord(
|
||||
TOPIC_SRC_SERVER_LOG_DLQ,
|
||||
"unknown",
|
||||
errorMessage,
|
||||
)
|
||||
|
||||
syslog.copyHeadersTo(record)
|
||||
|
||||
dlqProducer.send(record)
|
||||
}
|
||||
|
||||
|
||||
override fun close() {
|
||||
producer.close()
|
||||
}
|
||||
|
||||
|
||||
companion object {
|
||||
private fun log(msg: String) = println("[KafkatorioPacketKafkaProducer] $msg")
|
||||
|
||||
|
||||
private fun Properties.updateValue(
|
||||
key: String,
|
||||
update: (oldValue: String) -> String
|
||||
) {
|
||||
val newValue = update(getValue(key) as String)
|
||||
setProperty(key, newValue)
|
||||
}
|
||||
|
||||
|
||||
private fun SyslogMsg.copyHeadersTo(record: ProducerRecord<*, *>) {
|
||||
val headers = SyslogFieldKeys.values()
|
||||
.filter { "HEADER_" in it.name }
|
||||
.mapNotNull { headerKey ->
|
||||
val headerValue = src[headerKey.field]
|
||||
when {
|
||||
headerValue.isNullOrBlank() -> null
|
||||
else -> RecordHeader(
|
||||
headerKey.field,
|
||||
headerValue.toByteArray()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
headers.forEach { header ->
|
||||
record.headers().add(header)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,243 +1,243 @@
|
|||
package dev.adamko.kafkatorio.server.processor
|
||||
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
|
||||
import dev.adamko.kafkatorio.server.processor.topology.broadcastToWebsocket
|
||||
import dev.adamko.kafkatorio.server.processor.topology.colourEntityChunks
|
||||
import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
|
||||
import dev.adamko.kafkatorio.server.processor.topology.convertEntityUpdateToServerMapChunkTiles
|
||||
import dev.adamko.kafkatorio.server.processor.topology.convertResourceUpdateToServerMapChunkTiles
|
||||
import dev.adamko.kafkatorio.server.processor.topology.factorioServerPacketStream
|
||||
import dev.adamko.kafkatorio.server.processor.topology.saveMapTiles
|
||||
import dev.adamko.kafkatorio.server.processor.topology.splitFactorioServerPacketStream
|
||||
import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
|
||||
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
|
||||
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
import java.util.Properties
|
||||
import kotlin.coroutines.resume
|
||||
import kotlin.coroutines.resumeWithException
|
||||
import kotlin.io.path.Path
|
||||
import kotlin.time.Duration.Companion.milliseconds
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import kotlinx.coroutines.CoroutineName
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.currentCoroutineContext
|
||||
import kotlinx.coroutines.job
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
import org.apache.kafka.clients.producer.ProducerConfig
|
||||
import org.apache.kafka.streams.KafkaStreams
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.StreamsConfig
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.TopologyDescription
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
//import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
//import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
//import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
//import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
//import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
|
||||
//import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
|
||||
//import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
//import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.broadcastToWebsocket
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.colourEntityChunks
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.colourMapChunks
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.convertEntityUpdateToServerMapChunkTiles
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.convertResourceUpdateToServerMapChunkTiles
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.factorioServerPacketStream
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.saveMapTiles
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.splitFactorioServerPacketStream
|
||||
//import dev.adamko.kafkatorio.server.processor.topology.streamPacketData
|
||||
//import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
|
||||
//import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
//import java.util.Properties
|
||||
//import kotlin.coroutines.resume
|
||||
//import kotlin.coroutines.resumeWithException
|
||||
//import kotlin.io.path.Path
|
||||
//import kotlin.time.Duration.Companion.milliseconds
|
||||
//import kotlin.time.Duration.Companion.seconds
|
||||
//import kotlinx.coroutines.CoroutineName
|
||||
//import kotlinx.coroutines.CoroutineScope
|
||||
//import kotlinx.coroutines.cancel
|
||||
//import kotlinx.coroutines.currentCoroutineContext
|
||||
//import kotlinx.coroutines.job
|
||||
//import kotlinx.coroutines.launch
|
||||
//import kotlinx.coroutines.suspendCancellableCoroutine
|
||||
//import org.apache.kafka.clients.producer.ProducerConfig
|
||||
//import org.apache.kafka.streams.KafkaStreams
|
||||
//import org.apache.kafka.streams.StreamsBuilder
|
||||
//import org.apache.kafka.streams.StreamsConfig
|
||||
//import org.apache.kafka.streams.Topology
|
||||
//import org.apache.kafka.streams.TopologyDescription
|
||||
//import org.apache.kafka.streams.kstream.KStream
|
||||
|
||||
|
||||
internal class KafkatorioTopology(
|
||||
private val appProps: ApplicationProperties,
|
||||
private val wsServer: WebmapWebsocketServer,
|
||||
private val syslogServer: SyslogSocketServer,
|
||||
private val kafkatorioPacketKafkaProducer: KafkatorioPacketKafkaProducer,
|
||||
// private val appProps: ApplicationProperties,
|
||||
// private val wsServer: WebmapWebsocketServer,
|
||||
// private val syslogServer: SyslogSocketServer,
|
||||
// private val kafkatorioPacketKafkaProducer: KafkatorioPacketKafkaProducer,
|
||||
) {
|
||||
|
||||
private val builderScope = CoroutineScope(CoroutineName("KafkaTopology"))
|
||||
// private val builderScope = CoroutineScope(CoroutineName("KafkaTopology"))
|
||||
|
||||
|
||||
suspend fun start() = builderScope.launch {
|
||||
splitPackets()
|
||||
|
||||
groupTilesMapChunks()
|
||||
|
||||
groupEntitiesMapChunks(
|
||||
layer = ServerMapTileLayer.BUILDING,
|
||||
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES,
|
||||
colouredChunkStateTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
|
||||
)
|
||||
|
||||
groupEntitiesMapChunks(
|
||||
layer = ServerMapTileLayer.RESOURCE,
|
||||
colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES,
|
||||
colouredChunkStateTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
|
||||
)
|
||||
|
||||
saveTiles()
|
||||
|
||||
websocketBroadcaster()
|
||||
|
||||
syslogProducer()
|
||||
}
|
||||
// suspend fun start() = builderScope.launch {
|
||||
// splitPackets()
|
||||
//
|
||||
// groupTilesMapChunks()
|
||||
//
|
||||
//// groupEntitiesMapChunks(
|
||||
//// layer = ServerMapTileLayer.BUILDING,
|
||||
//// colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES,
|
||||
//// colouredChunkStateTopic = TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
|
||||
//// )
|
||||
//
|
||||
//// groupEntitiesMapChunks(
|
||||
//// layer = ServerMapTileLayer.RESOURCE,
|
||||
//// colouredUpdatesStreamTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES,
|
||||
//// colouredChunkStateTopic = TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
|
||||
//// )
|
||||
//
|
||||
// saveTiles()
|
||||
//
|
||||
// websocketBroadcaster()
|
||||
//
|
||||
// syslogProducer()
|
||||
// }
|
||||
|
||||
|
||||
private suspend fun splitPackets() {
|
||||
val builder = StreamsBuilder()
|
||||
val packets = factorioServerPacketStream(builder)
|
||||
splitFactorioServerPacketStream(packets)
|
||||
|
||||
val topology = builder.build()
|
||||
|
||||
launchTopology("splitPackets", topology)
|
||||
}
|
||||
// private suspend fun splitPackets() {
|
||||
// val builder = StreamsBuilder()
|
||||
// val packets = factorioServerPacketStream(builder)
|
||||
// splitFactorioServerPacketStream(packets)
|
||||
//
|
||||
// val topology = builder.build()
|
||||
//
|
||||
// launchTopology("splitPackets", topology)
|
||||
// }
|
||||
|
||||
|
||||
/** Terrain tiles */
|
||||
private suspend fun groupTilesMapChunks() {
|
||||
val builder = StreamsBuilder()
|
||||
// /** Terrain tiles */
|
||||
// private suspend fun groupTilesMapChunks() {
|
||||
// val builder = StreamsBuilder()
|
||||
//
|
||||
// val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
|
||||
// builder.streamPacketData()
|
||||
//
|
||||
// val topology = colourMapChunks(builder, protosStream)
|
||||
//
|
||||
// launchTopology("groupTilesMapChunks.${ServerMapTileLayer.TERRAIN.dir}", topology)
|
||||
// }
|
||||
|
||||
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
|
||||
builder.streamPacketData()
|
||||
|
||||
val topology = colourMapChunks(builder, protosStream)
|
||||
|
||||
launchTopology("groupTilesMapChunks.${ServerMapTileLayer.TERRAIN.dir}", topology)
|
||||
}
|
||||
//
|
||||
// /** Building and resource tiles */
|
||||
// private suspend fun groupEntitiesMapChunks(
|
||||
// layer: ServerMapTileLayer,
|
||||
// colouredUpdatesStreamTopic: String,
|
||||
// colouredChunkStateTopic: String,
|
||||
// ) {
|
||||
// val builder = StreamsBuilder()
|
||||
//
|
||||
// val entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> =
|
||||
// when (layer) {
|
||||
// ServerMapTileLayer.TERRAIN -> return
|
||||
//
|
||||
// ServerMapTileLayer.RESOURCE ->
|
||||
// builder.streamPacketData<MapChunkResourceUpdate>()
|
||||
// .convertResourceUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
|
||||
//
|
||||
// ServerMapTileLayer.BUILDING ->
|
||||
// builder.streamPacketData<MapChunkEntityUpdate>()
|
||||
// .convertEntityUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
|
||||
// }
|
||||
//
|
||||
// val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
|
||||
// builder.streamPacketData()
|
||||
//
|
||||
// val topology = colourEntityChunks(
|
||||
// builder,
|
||||
// entityChunksStream,
|
||||
// protosStream,
|
||||
// layer,
|
||||
// colouredUpdatesStreamTopic,
|
||||
// colouredChunkStateTopic,
|
||||
// )
|
||||
//
|
||||
// launchTopology("groupTilesMapChunks.${layer.dir}", topology)
|
||||
// }
|
||||
|
||||
|
||||
/** Building and resource tiles */
|
||||
private suspend fun groupEntitiesMapChunks(
|
||||
layer: ServerMapTileLayer,
|
||||
colouredUpdatesStreamTopic: String,
|
||||
colouredChunkStateTopic: String,
|
||||
) {
|
||||
val builder = StreamsBuilder()
|
||||
|
||||
val entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> =
|
||||
when (layer) {
|
||||
ServerMapTileLayer.TERRAIN -> return
|
||||
|
||||
ServerMapTileLayer.RESOURCE ->
|
||||
builder.streamPacketData<MapChunkResourceUpdate>()
|
||||
.convertResourceUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
|
||||
|
||||
ServerMapTileLayer.BUILDING ->
|
||||
builder.streamPacketData<MapChunkEntityUpdate>()
|
||||
.convertEntityUpdateToServerMapChunkTiles("groupEntitiesMapChunks.${layer.dir}")
|
||||
}
|
||||
|
||||
val protosStream: KStream<FactorioServerId, PrototypesUpdate> =
|
||||
builder.streamPacketData()
|
||||
|
||||
val topology = colourEntityChunks(
|
||||
builder,
|
||||
entityChunksStream,
|
||||
protosStream,
|
||||
layer,
|
||||
colouredUpdatesStreamTopic,
|
||||
colouredChunkStateTopic,
|
||||
)
|
||||
|
||||
launchTopology("groupTilesMapChunks.${layer.dir}", topology)
|
||||
}
|
||||
// private suspend fun saveTiles() {
|
||||
// val builder = StreamsBuilder()
|
||||
// val topology = saveMapTiles(builder, appProps.serverDataDir)
|
||||
// launchTopology("saveTiles", topology)
|
||||
// }
|
||||
|
||||
|
||||
private suspend fun saveTiles() {
|
||||
val builder = StreamsBuilder()
|
||||
val topology = saveMapTiles(builder, appProps.serverDataDir)
|
||||
launchTopology("saveTiles", topology)
|
||||
}
|
||||
// private suspend fun websocketBroadcaster() {
|
||||
// val builder = StreamsBuilder()
|
||||
// broadcastToWebsocket(wsServer, builder)
|
||||
//
|
||||
// val topology = builder.build()
|
||||
//
|
||||
// launchTopology(
|
||||
// "websocketBroadcaster", topology, mapOf(
|
||||
// StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
|
||||
// StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
|
||||
// )
|
||||
// )
|
||||
// }
|
||||
|
||||
|
||||
private suspend fun websocketBroadcaster() {
|
||||
val builder = StreamsBuilder()
|
||||
broadcastToWebsocket(wsServer, builder)
|
||||
|
||||
val topology = builder.build()
|
||||
|
||||
launchTopology(
|
||||
"websocketBroadcaster", topology, mapOf(
|
||||
StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
|
||||
StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
|
||||
)
|
||||
)
|
||||
}
|
||||
// /** Receive log messages from the syslog socket, and send them to a Kafka topic */
|
||||
// private fun syslogProducer() {
|
||||
// builderScope.launch {
|
||||
// kafkatorioPacketKafkaProducer.launch()
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
/** Receive log messages from the syslog socket, and send them to a Kafka topic */
|
||||
private fun syslogProducer() {
|
||||
builderScope.launch {
|
||||
kafkatorioPacketKafkaProducer.launch()
|
||||
}
|
||||
}
|
||||
// private suspend fun launchTopology(
|
||||
// id: String,
|
||||
// topology: Topology,
|
||||
// additionalProperties: Map<String, String> = mapOf(),
|
||||
// ) {
|
||||
//
|
||||
// val props: Properties = appProps.kafkaStreamsConfig.toProperties()
|
||||
//
|
||||
// additionalProperties.forEach { (k, v) -> props.setProperty(k, v) }
|
||||
//
|
||||
// val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
|
||||
// props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
|
||||
//
|
||||
// builderScope.launch(CoroutineName(appId)) {
|
||||
//
|
||||
// val streamsState: KafkaStreams.State = suspendCancellableCoroutine { cont ->
|
||||
//
|
||||
// val streams = KafkaStreams(topology, props)
|
||||
// streams.setUncaughtExceptionHandler(StreamsExceptionHandler { e ->
|
||||
// cont.resumeWithException(e)
|
||||
// })
|
||||
//
|
||||
// streams.setStateListener { newState: KafkaStreams.State, _ ->
|
||||
// when (newState) {
|
||||
// KafkaStreams.State.NOT_RUNNING,
|
||||
// KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
|
||||
//
|
||||
// else -> Unit // do nothing
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// cont.invokeOnCancellation {
|
||||
// println("[$appId] coroutine cancelled -> closing. cause:$it")
|
||||
// streams.close()
|
||||
// }
|
||||
//
|
||||
// printTopologyDescription(appId, topology)
|
||||
// streams.start()
|
||||
// println("launched Topology $appId")
|
||||
// }
|
||||
// println("exiting Topology $appId: $streamsState")
|
||||
// currentCoroutineContext().job.cancel("$appId: $streamsState")
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
private suspend fun launchTopology(
|
||||
id: String,
|
||||
topology: Topology,
|
||||
additionalProperties: Map<String, String> = mapOf(),
|
||||
) {
|
||||
|
||||
val props: Properties = appProps.kafkaStreamsConfig.toProperties()
|
||||
|
||||
additionalProperties.forEach { (k, v) -> props.setProperty(k, v) }
|
||||
|
||||
val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
|
||||
props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
|
||||
|
||||
builderScope.launch(CoroutineName(appId)) {
|
||||
|
||||
val streamsState: KafkaStreams.State = suspendCancellableCoroutine { cont ->
|
||||
|
||||
val streams = KafkaStreams(topology, props)
|
||||
streams.setUncaughtExceptionHandler(StreamsExceptionHandler { e ->
|
||||
cont.resumeWithException(e)
|
||||
})
|
||||
|
||||
streams.setStateListener { newState: KafkaStreams.State, _ ->
|
||||
when (newState) {
|
||||
KafkaStreams.State.NOT_RUNNING,
|
||||
KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
|
||||
|
||||
else -> Unit // do nothing
|
||||
}
|
||||
}
|
||||
|
||||
cont.invokeOnCancellation {
|
||||
println("[$appId] coroutine cancelled -> closing. cause:$it")
|
||||
streams.close()
|
||||
}
|
||||
|
||||
printTopologyDescription(appId, topology)
|
||||
streams.start()
|
||||
println("launched Topology $appId")
|
||||
}
|
||||
println("exiting Topology $appId: $streamsState")
|
||||
currentCoroutineContext().job.cancel("$appId: $streamsState")
|
||||
}
|
||||
}
|
||||
// private fun printTopologyDescription(appId: String, topology: Topology) {
|
||||
// val description: TopologyDescription = topology.describe()
|
||||
//
|
||||
// val descFile = Path("./build/$appId.txt").toFile().apply {
|
||||
// parentFile.mkdirs()
|
||||
// if (!exists()) createNewFile()
|
||||
// writeText(description.toString())
|
||||
// }
|
||||
//
|
||||
// println(
|
||||
// """
|
||||
// |----------------
|
||||
// |$appId - ${descFile.canonicalPath}
|
||||
// |$description
|
||||
// |----------------
|
||||
// """.trimMargin()
|
||||
// )
|
||||
// }
|
||||
|
||||
|
||||
private fun printTopologyDescription(appId: String, topology: Topology) {
|
||||
val description: TopologyDescription = topology.describe()
|
||||
|
||||
val descFile = Path("./build/$appId.txt").toFile().apply {
|
||||
parentFile.mkdirs()
|
||||
if (!exists()) createNewFile()
|
||||
writeText(description.toString())
|
||||
}
|
||||
|
||||
println(
|
||||
"""
|
||||
|----------------
|
||||
|$appId - ${descFile.canonicalPath}
|
||||
|$description
|
||||
|----------------
|
||||
""".trimMargin()
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
companion object {
|
||||
private fun Map<String, String>.toProperties(): Properties =
|
||||
entries.fold(Properties()) { props, (k, v) ->
|
||||
props.apply { setProperty(k, v) }
|
||||
}
|
||||
}
|
||||
// companion object {
|
||||
// private fun Map<String, String>.toProperties(): Properties =
|
||||
// entries.fold(Properties()) { props, (k, v) ->
|
||||
// props.apply { setProperty(k, v) }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor
|
||||
|
||||
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler
|
||||
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse
|
||||
|
||||
|
||||
class StreamsExceptionHandler(
|
||||
val shutdown: (e: Throwable) -> Unit,
|
||||
) : StreamsUncaughtExceptionHandler {
|
||||
override fun handle(exception: Throwable): StreamThreadExceptionResponse {
|
||||
exception.printStackTrace()
|
||||
shutdown(exception)
|
||||
return StreamThreadExceptionResponse.SHUTDOWN_APPLICATION
|
||||
}
|
||||
}
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.admin
|
||||
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.processor.allTopics
|
||||
import java.util.concurrent.TimeUnit
|
||||
import mu.KotlinLogging
|
||||
import org.apache.kafka.clients.admin.Admin
|
||||
import org.apache.kafka.clients.admin.CreateTopicsResult
|
||||
import org.apache.kafka.clients.admin.NewTopic
|
||||
import org.apache.kafka.clients.admin.TopicListing
|
||||
|
||||
|
||||
class KafkatorioKafkaAdmin(
|
||||
appProps: ApplicationProperties
|
||||
) {
|
||||
|
||||
private val logger = KotlinLogging.logger {}
|
||||
|
||||
init {
|
||||
println("Creating KafkatorioKafkaAdmin with config: ${appProps.kafkaAdminConfig}")
|
||||
}
|
||||
|
||||
private val kafkaAdmin: Admin = Admin.create(appProps.kafkaAdminConfig)
|
||||
|
||||
fun createKafkatorioTopics() {
|
||||
|
||||
val currentTopics = currentTopics().map { it.name() }
|
||||
|
||||
val kafkatorioTopics = allTopics() - currentTopics.toSet()
|
||||
|
||||
if (kafkatorioTopics.isNotEmpty()) {
|
||||
logger.info("Creating ${kafkatorioTopics.size} topics")
|
||||
val result = kafkaAdmin.createTopics { kafkatorioTopics }
|
||||
|
||||
// wait for all brokers to become aware of the created topics
|
||||
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
||||
|
||||
result.values().forEach {
|
||||
logger.info("Created topic: ${it.key} ${it.value.get()}")
|
||||
}
|
||||
|
||||
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
||||
|
||||
logger.info("Listing topics")
|
||||
kafkaAdmin.listTopics()
|
||||
.listings()
|
||||
.get()
|
||||
.forEach {
|
||||
logger.info("Topic: $it")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun Admin.createTopics(
|
||||
numPartitions: Int = 3,
|
||||
replicationFactor: Short = 1,
|
||||
topicNames: () -> Collection<String>,
|
||||
): CreateTopicsResult =
|
||||
createTopics(
|
||||
topicNames()
|
||||
.distinct()
|
||||
.map {
|
||||
NewTopic(it, numPartitions, replicationFactor)
|
||||
// .configs(mapOf("cleanup.policy" to "compact"))
|
||||
}
|
||||
)
|
||||
|
||||
private fun currentTopics(): MutableCollection<TopicListing> {
|
||||
return kafkaAdmin.listTopics()
|
||||
.listings()
|
||||
.get()
|
||||
}
|
||||
|
||||
// fun topicNameBase(packetType: KafkatorioPacketDataType): String =
|
||||
// "kafkatorio.${packetType.name}"
|
||||
}
|
||||
|
|
@ -1,35 +1,35 @@
|
|||
package dev.adamko.kafkatorio.server.processor
|
||||
|
||||
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.processor.admin.KafkatorioKafkaAdmin
|
||||
import dev.adamko.kafkatorio.server.security.Authenticator
|
||||
import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
|
||||
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
import kotlinx.coroutines.awaitCancellation
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
|
||||
suspend fun eventsProcessor(
|
||||
appProps: ApplicationProperties,
|
||||
syslogServer: SyslogSocketServer,
|
||||
wsServer: WebmapWebsocketServer,
|
||||
authenticator: Authenticator,
|
||||
): Unit = coroutineScope {
|
||||
|
||||
val admin = KafkatorioKafkaAdmin(appProps)
|
||||
admin.createKafkatorioTopics()
|
||||
|
||||
val kafkatorioPacketKafkaProducer = KafkatorioPacketKafkaProducer(appProps, syslogServer, authenticator)
|
||||
|
||||
val topology = KafkatorioTopology(appProps, wsServer, syslogServer, kafkatorioPacketKafkaProducer)
|
||||
|
||||
launch {
|
||||
topology.start()
|
||||
|
||||
println("launched KafkatorioTopology")
|
||||
|
||||
awaitCancellation()
|
||||
}
|
||||
}
|
||||
//import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
//import dev.adamko.kafkatorio.server.processor.admin.KafkatorioKafkaAdmin
|
||||
//import dev.adamko.kafkatorio.server.security.Authenticator
|
||||
//import dev.adamko.kafkatorio.server.socket.SyslogSocketServer
|
||||
//import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
//import kotlinx.coroutines.awaitCancellation
|
||||
//import kotlinx.coroutines.coroutineScope
|
||||
//import kotlinx.coroutines.launch
|
||||
//
|
||||
//
|
||||
//suspend fun eventsProcessor(
|
||||
// appProps: ApplicationProperties,
|
||||
// syslogServer: SyslogSocketServer,
|
||||
// wsServer: WebmapWebsocketServer,
|
||||
// authenticator: Authenticator,
|
||||
//): Unit = coroutineScope {
|
||||
//
|
||||
// val admin = KafkatorioKafkaAdmin(appProps)
|
||||
// admin.createKafkatorioTopics()
|
||||
//
|
||||
// val kafkatorioPacketKafkaProducer = KafkatorioPacketKafkaProducer(appProps, syslogServer, authenticator)
|
||||
//
|
||||
// val topology = KafkatorioTopology(appProps, wsServer, syslogServer, kafkatorioPacketKafkaProducer)
|
||||
//
|
||||
// launch {
|
||||
// topology.start()
|
||||
//
|
||||
// println("launched KafkatorioTopology")
|
||||
//
|
||||
// awaitCancellation()
|
||||
// }
|
||||
//}
|
||||
|
|
|
|||
|
|
@ -1,99 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor
|
||||
|
||||
import dev.adamko.kafkatorio.schema.packets.ConfigurationUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.EntityUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.SurfaceUpdate
|
||||
import kotlin.reflect.KClass
|
||||
|
||||
|
||||
private const val DOMAIN = "kafkatorio"
|
||||
|
||||
const val TOPIC_SRC_SERVER_LOG = "$DOMAIN.src.server-log"
|
||||
const val TOPIC_SRC_SERVER_LOG_DLQ = "$DOMAIN.src.server-log.dlq"
|
||||
|
||||
//const val TOPIC_GROUPED_MAP_CHUNKS_STATE = "$DOMAIN.state.map-chunks.grouped"
|
||||
|
||||
const val TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES =
|
||||
"$DOMAIN.state-updates.map-chunk-032.terrain.colour"
|
||||
const val TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES =
|
||||
"$DOMAIN.state-updates.map-chunk-032.resource.colour"
|
||||
const val TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES =
|
||||
"$DOMAIN.state-updates.map-chunk-032.building.colour"
|
||||
|
||||
const val TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE = "$DOMAIN.state.map-chunk.terrain.colour"
|
||||
const val TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE = "$DOMAIN.state.map-chunk.resource.colour"
|
||||
const val TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE = "$DOMAIN.state.map-chunk.building.colour"
|
||||
|
||||
//const val TOPIC_SUBDIVIDED_MAP_TILES = "$DOMAIN.map-tiles.subdivided"
|
||||
//const val TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED = "$TOPIC_SUBDIVIDED_MAP_TILES.debounced"
|
||||
|
||||
|
||||
/** Broadcast these events out, via websocket */
|
||||
const val TOPIC_BROADCAST_TO_WEBSOCKET = "$DOMAIN.broadcast.websocket"
|
||||
|
||||
fun allTopics(): Set<String> = buildSet {
|
||||
add(TOPIC_SRC_SERVER_LOG)
|
||||
add(TOPIC_SRC_SERVER_LOG_DLQ)
|
||||
|
||||
// add(TOPIC_GROUPED_MAP_CHUNKS_STATE)
|
||||
|
||||
add(TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES)
|
||||
add(TOPIC_MAP_CHUNK_RESOURCE_COLOURED_032_UPDATES)
|
||||
add(TOPIC_MAP_CHUNK_BUILDING_COLOURED_032_UPDATES)
|
||||
|
||||
add(TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE)
|
||||
add(TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE)
|
||||
add(TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE)
|
||||
|
||||
// add(TOPIC_SUBDIVIDED_MAP_TILES)
|
||||
// add(TOPIC_SUBDIVIDED_MAP_TILES_DEBOUNCED)
|
||||
|
||||
add(TOPIC_BROADCAST_TO_WEBSOCKET)
|
||||
|
||||
addAll(packetTopicNames.values)
|
||||
}
|
||||
|
||||
val KafkatorioPacketData.topicName: String
|
||||
get() = when (this) {
|
||||
is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class)
|
||||
is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class)
|
||||
is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class)
|
||||
is PrototypesUpdate -> packetTopicNames.getValue(PrototypesUpdate::class)
|
||||
is SurfaceUpdate -> packetTopicNames.getValue(SurfaceUpdate::class)
|
||||
is EntityUpdate -> packetTopicNames.getValue(EntityUpdate::class)
|
||||
is MapChunkTileUpdate -> packetTopicNames.getValue(MapChunkTileUpdate::class)
|
||||
is PlayerUpdate -> packetTopicNames.getValue(PlayerUpdate::class)
|
||||
is KafkatorioPacketDataError -> packetTopicNames.getValue(KafkatorioPacketDataError::class)
|
||||
is MapChunkEntityUpdate -> packetTopicNames.getValue(MapChunkEntityUpdate::class)
|
||||
is MapChunkResourceUpdate -> packetTopicNames.getValue(MapChunkResourceUpdate::class)
|
||||
}
|
||||
|
||||
|
||||
private val packetTopicNames: Map<KClass<out KafkatorioPacketData>, String> = mapOf(
|
||||
ConfigurationUpdate::class to "configuration",
|
||||
ConsoleChatUpdate::class to "console-chat",
|
||||
ConsoleCommandUpdate::class to "console-command",
|
||||
PrototypesUpdate::class to "prototypes",
|
||||
SurfaceUpdate::class to "surface",
|
||||
EntityUpdate::class to "entity",
|
||||
MapChunkTileUpdate::class to "map-chunk-tile",
|
||||
PlayerUpdate::class to "player",
|
||||
MapChunkEntityUpdate::class to "map-chunk-entity",
|
||||
MapChunkResourceUpdate::class to "map-chunk-resource",
|
||||
KafkatorioPacketDataError::class to "error",
|
||||
).mapValues { (_, v) ->
|
||||
"$DOMAIN.packet.$v"
|
||||
}
|
||||
|
||||
|
||||
val KClass<out KafkatorioPacketData>.topicName: String
|
||||
get() = packetTopicNames.getValue(this)
|
||||
|
|
@ -1,329 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.library.resize
|
||||
import dev.adamko.kafkatorio.library.toMapChunkPosition
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kafkatorio.server.processor.DebounceProcessor.Companion.addDebounceProcessor
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.groupedAs
|
||||
import dev.adamko.kotka.extensions.materializedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.repartitionedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.flatMap
|
||||
import dev.adamko.kotka.extensions.streams.map
|
||||
import dev.adamko.kotka.extensions.streams.peek
|
||||
import dev.adamko.kotka.extensions.streams.reduce
|
||||
import dev.adamko.kotka.extensions.tableJoined
|
||||
import dev.adamko.kotka.extensions.tables.join
|
||||
import dev.adamko.kotka.extensions.tables.toStream
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
import org.apache.kafka.streams.kstream.KTable
|
||||
|
||||
|
||||
/** Add colour to each [MapChunkTileUpdate] */
|
||||
fun colourMapChunks(
|
||||
builder: StreamsBuilder,
|
||||
protosStream: KStream<FactorioServerId, PrototypesUpdate>
|
||||
): Topology {
|
||||
val pid = "colourMapChunks.terrain"
|
||||
|
||||
val mapChunksStream: KStream<FactorioServerId, MapChunkTileUpdate> = builder.streamPacketData()
|
||||
|
||||
val protosTable: KTable<FactorioServerId, TileColourDict> =
|
||||
protosStream.createTilePrototypeTable()
|
||||
|
||||
val chunkTilesColouredTable032: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
mapChunksStream
|
||||
.convertToServerMapChunkTiles(pid)
|
||||
.reduceServerMapChunkTilesToTable(pid)
|
||||
.enrichWithColourData(pid, protosTable)
|
||||
|
||||
chunkTilesColouredTable032.streamMapChunkColouredTo(
|
||||
"$pid.${ChunkSize.CHUNK_032}",
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES,
|
||||
)
|
||||
|
||||
val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
builder.stream(
|
||||
consumedAs(
|
||||
"$pid.consume.map-chunks-debounced",
|
||||
kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
),
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
|
||||
)
|
||||
|
||||
mapOf(
|
||||
// ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
|
||||
// ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
|
||||
// ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
|
||||
// ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
|
||||
ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
|
||||
ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
).forEach { (from, to) ->
|
||||
chunkTilesColoured
|
||||
.filter("$pid.combine-chunks.filter-$from-to-$to") { key: ServerMapChunkId, _: ServerMapChunkTiles<ColourHex> ->
|
||||
key.chunkSize == from
|
||||
}.groupByChunkPosition("$pid.from-$from-to-$to", to)
|
||||
.streamMapChunkColouredTo(
|
||||
"$pid.from-$from-to-$to",
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
|
||||
)
|
||||
}
|
||||
|
||||
return builder.build()
|
||||
.addDebounceProcessor(
|
||||
namePrefix = "groupMapChunks.terrain",
|
||||
sourceTopic = TOPIC_MAP_CHUNK_TERRAIN_COLOURED_032_UPDATES,
|
||||
sinkTopic = TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
|
||||
inactivityDuration = 15.seconds,
|
||||
keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
private fun KStream<FactorioServerId, MapChunkTileUpdate>.convertToServerMapChunkTiles(
|
||||
callerPid: String
|
||||
): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.convertToMapTiles"
|
||||
|
||||
return map("$pid.map") { serverId: FactorioServerId, update: MapChunkTileUpdate ->
|
||||
|
||||
val mapTileList = update.tileDictionary?.toMapTileList() ?: emptyList()
|
||||
|
||||
val (validMapTiles, invalidMapTiles) = mapTileList.partition {
|
||||
it.position in update.key.chunkPosition
|
||||
}
|
||||
|
||||
if (invalidMapTiles.isNotEmpty()) {
|
||||
println(
|
||||
"WARNING [$pid] MapChunkTileUpdate contained ${invalidMapTiles.size} out-of-bounds tiles, " +
|
||||
"key: ${update.key},\n" +
|
||||
"invalid tiles:" + invalidMapTiles.joinToString(limit = 10)
|
||||
)
|
||||
}
|
||||
|
||||
val validMapTilesToProtoHashCode = validMapTiles.associate {
|
||||
it.position to PrototypeHashCode(it.protoId)
|
||||
}
|
||||
|
||||
val chunkId = ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.TERRAIN,
|
||||
chunkPosition = update.key.chunkPosition,
|
||||
surfaceIndex = update.key.surfaceIndex,
|
||||
)
|
||||
|
||||
chunkId to ServerMapChunkTiles(chunkId, validMapTilesToProtoHashCode)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.reduceServerMapChunkTilesToTable(
|
||||
callerPid: String,
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.reduceServerMapChunkTilesToTable"
|
||||
|
||||
return repartition(
|
||||
repartitionedAs(
|
||||
"$pid.pre-table-repartition",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
|
||||
// force, otherwise KTable-KTable FK join doesn't work
|
||||
numberOfPartitions = 1,
|
||||
)
|
||||
).groupByKey(
|
||||
groupedAs(
|
||||
"$pid.group-by-key",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
|
||||
)
|
||||
).reduce(
|
||||
"$pid.reduce",
|
||||
materializedAs(
|
||||
"$pid.reduce.store",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<PrototypeHashCode>>(),
|
||||
)
|
||||
) { chunkTiles, otherChunkTiles ->
|
||||
chunkTiles + otherChunkTiles
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.enrichWithColourData(
|
||||
callerPid: String,
|
||||
protosTable: KTable<FactorioServerId, TileColourDict>,
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val pid = "$callerPid.enrichWithColourData"
|
||||
|
||||
return join(
|
||||
other = protosTable,
|
||||
tableJoined = tableJoined("$pid.join-tiles-with-prototypes"),
|
||||
materialized = materializedAs(
|
||||
"$pid.join-tiles-with-prototypes.store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
),
|
||||
foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<PrototypeHashCode> ->
|
||||
chunkTiles.chunkId.serverId
|
||||
}
|
||||
) { tiles: ServerMapChunkTiles<PrototypeHashCode>, colourDict: TileColourDict ->
|
||||
println("joining tiles:${tiles.map.size} with colourDict:${colourDict.size}")
|
||||
|
||||
val missingProtos = mutableSetOf<PrototypeHashCode>()
|
||||
|
||||
val tileColours: Map<MapTilePosition, ColourHex> =
|
||||
tiles.map.mapNotNull { (position, code) ->
|
||||
when (val colour = colourDict[code]) {
|
||||
null -> {
|
||||
missingProtos.add(code)
|
||||
null
|
||||
}
|
||||
else -> position to colour
|
||||
}
|
||||
}.toMap()
|
||||
|
||||
if (missingProtos.isNotEmpty()) {
|
||||
println("[$pid] missing ${missingProtos.size} tile prototypes: ${missingProtos.joinToString { "${it.code}" }}")
|
||||
}
|
||||
|
||||
println("[$pid] Set ${tileColours.size} tile colours for chunk ${tiles.chunkId}")
|
||||
|
||||
ServerMapChunkTiles(tiles.chunkId, tileColours)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.streamMapChunkColouredTo(
|
||||
callerPid: String,
|
||||
outputTopic: String,
|
||||
) {
|
||||
val pid = "$callerPid.output-chunk"
|
||||
|
||||
toStream("$pid.stream")
|
||||
.flatMap("$pid.group-by-chunks") { _: ServerMapChunkId, tiles: ServerMapChunkTiles<ColourHex>? ->
|
||||
reChunkTiles(pid, tiles)
|
||||
}.peek("$pid.print-group-result") { _, chunkTiles ->
|
||||
println("[$pid] Grouping map tiles result: ${chunkTiles.chunkId} / size:${chunkTiles.map.size}")
|
||||
}.to(
|
||||
outputTopic,
|
||||
producedAs(
|
||||
"$pid.grouped-map-chunks",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Group [MapChunkTileUpdate]s by the Chunk position.
|
||||
*
|
||||
* They should already be grouped, but do it again to make sure, and to filter out empty updates.
|
||||
*/
|
||||
fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.groupByChunkPosition(
|
||||
callerPid: String,
|
||||
targetChunkSize: ChunkSize = ChunkSize.CHUNK_032
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val pid = "$callerPid.groupByChunkPosition.${targetChunkSize.name}"
|
||||
|
||||
return flatMap("$pid.change-chunk-size") { _: ServerMapChunkId, chunkTiles: ServerMapChunkTiles<ColourHex>? ->
|
||||
// println("[$pid] changing chunk size for $chunkId")
|
||||
chunkTiles ?: return@flatMap emptyList()
|
||||
|
||||
val newChunkPosition = chunkTiles.chunkId.chunkPosition.resize(targetChunkSize)
|
||||
val newId = chunkTiles.chunkId.copy(chunkPosition = newChunkPosition)
|
||||
val newChunkTiles = chunkTiles.copy(chunkId = newId)
|
||||
|
||||
reChunkTiles(pid, newChunkTiles)
|
||||
}.groupByKey(
|
||||
groupedAs(
|
||||
"$pid.group-by-key",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
|
||||
)
|
||||
).reduce(
|
||||
"$pid.reduce",
|
||||
materializedAs(
|
||||
"$pid.reduce.store",
|
||||
kxsBinary.serde<ServerMapChunkId>(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
|
||||
)
|
||||
) { chunkTiles, otherChunkTiles ->
|
||||
chunkTiles + otherChunkTiles
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun <T : Any> reChunkTiles(
|
||||
pid: String,
|
||||
tiles: ServerMapChunkTiles<T>?,
|
||||
): List<Pair<ServerMapChunkId, ServerMapChunkTiles<T>>> {
|
||||
|
||||
return when {
|
||||
tiles == null || tiles.map.isEmpty() -> emptyList()
|
||||
|
||||
tiles.map.keys.all { it in tiles.chunkId.chunkPosition } -> listOf(tiles.chunkId to tiles)
|
||||
|
||||
else -> {
|
||||
val chunkId = tiles.chunkId
|
||||
|
||||
val (boundedTiles, unboundedTiles) = tiles.map.entries.partition { (position, _) ->
|
||||
position in chunkId.chunkPosition
|
||||
}
|
||||
|
||||
if (unboundedTiles.isNotEmpty()) {
|
||||
val unboundedEntities = unboundedTiles.map { (_, entity) -> entity }
|
||||
println(
|
||||
"WARNING [$pid] MapChunkEntityUpdate $chunkId "
|
||||
+ "contained ${unboundedEntities.size} out-of-bounds entities "
|
||||
+ unboundedEntities.joinToString(limit = 10)
|
||||
)
|
||||
}
|
||||
|
||||
// the existing chunkId and its tiles
|
||||
val boundedTilesChunk =
|
||||
chunkId to ServerMapChunkTiles(chunkId, boundedTiles.associate { it.key to it.value })
|
||||
|
||||
// re-calculate a chunk for the out-of-bounds tiles
|
||||
val chunkedUnboundedTiles = unboundedTiles
|
||||
.map { it.key to it.value }
|
||||
.groupBy { (position, _) ->
|
||||
position.toMapChunkPosition(chunkId.chunkSize)
|
||||
}.map { (newChunkPosition, tiles) ->
|
||||
val newChunkId = chunkId.copy(chunkPosition = newChunkPosition)
|
||||
newChunkId to ServerMapChunkTiles(newChunkId, tiles.toMap())
|
||||
}
|
||||
|
||||
println("[$pid] re-bounded ${unboundedTiles.size} tiles into ${chunkedUnboundedTiles.size} chunks in $chunkId")
|
||||
|
||||
listOf(boundedTilesChunk) + chunkedUnboundedTiles
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,270 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.library.toMapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioPrototype
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapBoundingBox
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.common.toHex
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkEntityUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkResourceUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.MapChunkTileUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kafkatorio.server.processor.DebounceProcessor.Companion.addDebounceProcessor
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.materializedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.flatMap
|
||||
import dev.adamko.kotka.extensions.tableJoined
|
||||
import dev.adamko.kotka.extensions.tables.join
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.math.ceil
|
||||
import kotlin.math.floor
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
import org.apache.kafka.streams.kstream.KTable
|
||||
|
||||
|
||||
/**
|
||||
* Add colour to each [MapChunkTileUpdate]
|
||||
*
|
||||
* @param[colouredUpdatesStreamTopic] Output topic for the result of converting input updates into 32 sized chunks
|
||||
* @param[colouredChunkStateTopic] Post-debounce state topic, used to recursively combine chunks
|
||||
*/
|
||||
fun colourEntityChunks(
|
||||
builder: StreamsBuilder,
|
||||
entityChunksStream: KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>,
|
||||
protosStream: KStream<FactorioServerId, PrototypesUpdate>,
|
||||
layer: ServerMapTileLayer,
|
||||
colouredUpdatesStreamTopic: String,
|
||||
colouredChunkStateTopic: String,
|
||||
): Topology {
|
||||
val pid = "colourEntityChunks.${layer.dir}"
|
||||
|
||||
val protosTable: KTable<FactorioServerId, EntityPrototypeMap> =
|
||||
protosStream.createEntityPrototypeMap(pid)
|
||||
|
||||
val chunkEntitiesColouredTable032: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
entityChunksStream
|
||||
.reduceServerMapChunkTilesToTable(pid)
|
||||
.enrichWithColourData(pid, protosTable)
|
||||
|
||||
chunkEntitiesColouredTable032.streamMapChunkColouredTo(
|
||||
"$pid.${ChunkSize.CHUNK_032}",
|
||||
colouredUpdatesStreamTopic,
|
||||
)
|
||||
|
||||
val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
builder.stream(
|
||||
consumedAs(
|
||||
"$pid.consume.map-chunks-debounced",
|
||||
kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
),
|
||||
colouredChunkStateTopic,
|
||||
)
|
||||
|
||||
mapOf(
|
||||
// ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
|
||||
// ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
|
||||
// ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
|
||||
// ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
|
||||
ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
|
||||
ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
|
||||
).forEach { (from, to) ->
|
||||
chunkTilesColoured
|
||||
.filter("$pid.combine-chunks.filter-$from-to-$to") { key, _ ->
|
||||
key.chunkSize == from && key.layer == layer
|
||||
}.groupByChunkPosition(pid, to)
|
||||
.streamMapChunkColouredTo(
|
||||
"$pid.from-$from-to-$to",
|
||||
colouredChunkStateTopic,
|
||||
)
|
||||
}
|
||||
|
||||
return builder.build().addDebounceProcessor(
|
||||
namePrefix = "groupMapChunks.${layer.dir}",
|
||||
sourceTopic = colouredUpdatesStreamTopic,
|
||||
sinkTopic = colouredChunkStateTopic,
|
||||
inactivityDuration = 15.seconds,
|
||||
keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
fun KStream<FactorioServerId, MapChunkEntityUpdate>.convertEntityUpdateToServerMapChunkTiles(
|
||||
callerPid: String,
|
||||
): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.convertToServerMapChunkTiles"
|
||||
|
||||
return flatMap("$pid.map") { serverId: FactorioServerId, update: MapChunkEntityUpdate ->
|
||||
|
||||
//
|
||||
// // TODO There might be multiple resources per tile position, therefore this will arbitrarily
|
||||
// // exclude resources. Maybe change it from a Map to a List? Or have multiple resources per
|
||||
// // tile position?
|
||||
// val mapValidEntitiesToPrototypeHash = validEntityPositions.associate { (pos, resource) ->
|
||||
// pos to PrototypeHashCode(resource.protoId)
|
||||
// }
|
||||
|
||||
val entitiesToPrototypeHash = update.entities().associate { entity ->
|
||||
entity.position.toMapTilePosition() to PrototypeHashCode(entity.protoId)
|
||||
}
|
||||
|
||||
val chunkId = ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.BUILDING,
|
||||
chunkPosition = update.key.chunkPosition,
|
||||
surfaceIndex = update.key.surfaceIndex,
|
||||
)
|
||||
|
||||
val tiles = ServerMapChunkTiles(chunkId, entitiesToPrototypeHash)
|
||||
|
||||
reChunkTiles(pid, tiles)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun KStream<FactorioServerId, MapChunkResourceUpdate>.convertResourceUpdateToServerMapChunkTiles(
|
||||
callerPid: String,
|
||||
): KStream<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>> {
|
||||
|
||||
val pid = "$callerPid.convertResourceUpdateToChunkTiles"
|
||||
|
||||
return flatMap("$pid.map") { serverId: FactorioServerId, update: MapChunkResourceUpdate ->
|
||||
|
||||
// // TODO There might be multiple resources per tile position, therefore this will arbitrarily
|
||||
// // exclude resources. Maybe change it from a Map to a List? Or have multiple resources per
|
||||
// // tile position?
|
||||
// val mapValidEntitiesToPrototypeHash = validEntityPositions.associate { (pos, resource) ->
|
||||
// pos to PrototypeHashCode(resource.protoId)
|
||||
// }
|
||||
|
||||
val entitiesToPrototypeHash = update.resources().associate { entity ->
|
||||
entity.position.toMapTilePosition() to PrototypeHashCode(entity.protoId)
|
||||
}
|
||||
|
||||
val chunkId = ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.RESOURCE,
|
||||
chunkPosition = update.key.chunkPosition,
|
||||
surfaceIndex = update.key.surfaceIndex,
|
||||
)
|
||||
|
||||
val tiles = ServerMapChunkTiles(chunkId, entitiesToPrototypeHash)
|
||||
|
||||
reChunkTiles(pid, tiles)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun KTable<ServerMapChunkId, ServerMapChunkTiles<PrototypeHashCode>>.enrichWithColourData(
|
||||
callerPid: String,
|
||||
protosTable: KTable<FactorioServerId, EntityPrototypeMap>,
|
||||
): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val pid = "$callerPid.enrichWithColourData"
|
||||
|
||||
return join(
|
||||
other = protosTable,
|
||||
tableJoined = tableJoined("$pid.join-tiles-with-prototypes"),
|
||||
materialized = materializedAs(
|
||||
"$pid.join-tiles-with-prototypes.store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
),
|
||||
foreignKeyExtractor = { chunkTiles: ServerMapChunkTiles<PrototypeHashCode> ->
|
||||
chunkTiles.chunkId.serverId
|
||||
}) { tiles: ServerMapChunkTiles<PrototypeHashCode>, prototypes: EntityPrototypeMap ->
|
||||
println("joining tiles:${tiles.map.size} with prototypes:${prototypes.size}")
|
||||
|
||||
val missingProtos = mutableSetOf<PrototypeHashCode>()
|
||||
|
||||
val tileColours: Map<MapTilePosition, ColourHex> = buildList {
|
||||
tiles.map.forEach { (position, code) ->
|
||||
when (val proto = prototypes[code]) {
|
||||
|
||||
null -> missingProtos += code
|
||||
|
||||
else -> when (val box = proto.collisionBox) {
|
||||
|
||||
null -> add(position to proto.mapColour())
|
||||
|
||||
else -> box.iterator(position).forEach { entitySubTile ->
|
||||
add(entitySubTile to proto.mapColour())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}.fold(mapOf()) { acc, (pos, colour) ->
|
||||
// merge overlapping colours
|
||||
val newColour = when (val existing = acc[pos]) {
|
||||
null -> colour
|
||||
else -> (existing / 2u) + (colour / 2u)
|
||||
}
|
||||
acc + (pos to newColour)
|
||||
}
|
||||
|
||||
if (missingProtos.isNotEmpty()) {
|
||||
println("[$pid] missing ${missingProtos.size} tile prototypes: ${missingProtos.joinToString { "${it.code}" }}")
|
||||
}
|
||||
|
||||
println("[$pid] Set ${tileColours.size} tile colours for chunk ${tiles.chunkId}")
|
||||
|
||||
ServerMapChunkTiles(tiles.chunkId, tileColours)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun MapBoundingBox.iterator(origin: MapTilePosition): Iterator<MapTilePosition> {
|
||||
|
||||
val xOffset: Int = floor(tileWidth.toDouble() / 2).toInt()
|
||||
val startX = origin.x - xOffset
|
||||
val endX = origin.x + xOffset
|
||||
|
||||
val yOffset: Int = floor(tileHeight.toDouble() / 2).toInt()
|
||||
val startY = origin.y - yOffset
|
||||
val endY = origin.y + yOffset
|
||||
|
||||
return iterator {
|
||||
for (x in (startX..endX)) {
|
||||
for (y in (startY..endY)) { // screen coordinates = +y points to the floor, -y to the ceiling
|
||||
yield(MapTilePosition(x, y))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun FactorioPrototype.Entity.mapColour(): ColourHex =
|
||||
(colour ?: mapColour ?: mapColourFriend)?.toHex() ?: ColourHex.WHITE
|
||||
|
||||
|
||||
private operator fun ColourHex.div(divisor: UInt): ColourHex = copy(
|
||||
red = (red / divisor).toUByte(),
|
||||
green = (green / divisor).toUByte(),
|
||||
blue = (blue / divisor).toUByte(),
|
||||
alpha = (alpha / divisor).toUByte(),
|
||||
)
|
||||
|
||||
|
||||
private operator fun ColourHex.plus(other: ColourHex): ColourHex = copy(
|
||||
red = (red + other.red).toUByte(),
|
||||
green = (green + other.green).toUByte(),
|
||||
blue = (blue + other.blue).toUByte(),
|
||||
alpha = (alpha + other.alpha).toUByte(),
|
||||
)
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.Tick
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketDataError
|
||||
import dev.adamko.kafkatorio.server.config.jsonMapper
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_SRC_SERVER_LOG
|
||||
import dev.adamko.kafkatorio.server.processor.topicName
|
||||
import dev.adamko.kotka.extensions.component1
|
||||
import dev.adamko.kotka.extensions.component2
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.extensions.streams.to
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import java.util.zip.Inflater
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import okio.Buffer
|
||||
import okio.ByteString
|
||||
import okio.ByteString.Companion.decodeBase64
|
||||
import okio.buffer
|
||||
import okio.inflate
|
||||
import org.apache.kafka.common.serialization.Serdes
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
|
||||
|
||||
fun factorioServerPacketStream(
|
||||
builder: StreamsBuilder = StreamsBuilder(),
|
||||
): KStream<FactorioServerId, KafkatorioPacket> {
|
||||
|
||||
return builder.stream(
|
||||
TOPIC_SRC_SERVER_LOG,
|
||||
consumedAs(
|
||||
"read-raw-packets-from-server",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
Serdes.String(),
|
||||
)
|
||||
).mapValues("decode-packets") { _: FactorioServerId, value: String? ->
|
||||
|
||||
val packet = try {
|
||||
val decoded = decodeFactorioEncodedString(value!!)
|
||||
jsonMapper.decodeFromString(KafkatorioPacket.serializer(), decoded)
|
||||
} catch (e: Exception) {
|
||||
if (e is CancellationException) throw e
|
||||
|
||||
KafkatorioPacket(
|
||||
modVersion = "unknown",
|
||||
tick = Tick(0u),
|
||||
data = KafkatorioPacketDataError(
|
||||
exception = e,
|
||||
rawValue = value,
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
if (packet.data is KafkatorioPacketDataError) {
|
||||
println("error parsing $TOPIC_SRC_SERVER_LOG message: $packet")
|
||||
}
|
||||
|
||||
packet
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun splitFactorioServerPacketStream(
|
||||
factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>
|
||||
) {
|
||||
factorioServerPacketStream
|
||||
.to(
|
||||
producedAs(
|
||||
"split-server-log",
|
||||
jsonMapper.serde(),
|
||||
jsonMapper.serde(),
|
||||
)
|
||||
) { (_: FactorioServerId, value: KafkatorioPacket) ->
|
||||
// println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}")
|
||||
value.data.topicName
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Factorio `game.encode_string` zlib encodes a string, then base64's it. Here we do the reverse.
|
||||
*/
|
||||
private fun decodeFactorioEncodedString(
|
||||
source: String
|
||||
): String {
|
||||
val bytes: ByteString = source.decodeBase64() ?: error("failed to base64 decode $source")
|
||||
|
||||
return Buffer().write(bytes)
|
||||
.inflate(Inflater())
|
||||
.buffer()
|
||||
.readString(Charsets.UTF_8)
|
||||
}
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import com.sksamuel.scrimage.color.RGBColor
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.MapTile
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import java.awt.Color
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
|
||||
@Serializable
|
||||
data class ServerMapChunkTiles<Data : Any>(
|
||||
val chunkId: ServerMapChunkId,
|
||||
val map: Map<MapTilePosition, Data>,
|
||||
) {
|
||||
operator fun plus(other: ServerMapChunkTiles<Data>) = copy(map = map + other.map)
|
||||
}
|
||||
|
||||
|
||||
val MapTile.position
|
||||
get() = MapTilePosition(x = x, y = y)
|
||||
|
||||
|
||||
val TRANSPARENT_AWT: Color = ColourHex.TRANSPARENT.toRgbColor().awt()
|
||||
|
||||
|
||||
fun ColourHex.toRgbColor(): RGBColor {
|
||||
return RGBColor(
|
||||
red.toInt(),
|
||||
green.toInt(),
|
||||
blue.toInt(),
|
||||
alpha.toInt(),
|
||||
)
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
|
||||
import dev.adamko.kafkatorio.server.config.jsonMapper
|
||||
import dev.adamko.kafkatorio.server.processor.topicName
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
|
||||
|
||||
inline fun <reified T : KafkatorioPacketData> StreamsBuilder.streamPacketData(): KStream<FactorioServerId, T> {
|
||||
|
||||
val simpleName = T::class.simpleName!!
|
||||
val topicName = T::class.topicName
|
||||
|
||||
val taskName = "consume.${simpleName}"
|
||||
|
||||
return stream<FactorioServerId, KafkatorioPacket>(
|
||||
topicName,
|
||||
consumedAs("$taskName.input", jsonMapper.serde(), jsonMapper.serde())
|
||||
).filter("$taskName.filter-is-instance") { key: FactorioServerId?, value: KafkatorioPacket? ->
|
||||
key != null && value?.data is T
|
||||
}.mapValues("$taskName.map-values") { _: FactorioServerId, packet: KafkatorioPacket ->
|
||||
packet.data as T
|
||||
}
|
||||
}
|
||||
|
|
@ -1,138 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioEntityData
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioPrototype
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeHashCode
|
||||
import dev.adamko.kafkatorio.schema.common.PrototypeId
|
||||
import dev.adamko.kafkatorio.schema.common.toHex
|
||||
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
|
||||
import dev.adamko.kotka.extensions.groupedAs
|
||||
import dev.adamko.kotka.extensions.materializedAs
|
||||
import dev.adamko.kotka.extensions.materializedWith
|
||||
import dev.adamko.kotka.extensions.*
|
||||
import dev.adamko.kotka.extensions.streams.*
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.extensions.streams.toTable
|
||||
import dev.adamko.kotka.extensions.tables.*
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlinx.serialization.Serializable
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
import org.apache.kafka.streams.kstream.KTable
|
||||
|
||||
|
||||
@JvmInline
|
||||
@Serializable
|
||||
value class TileColourDict(
|
||||
private val map: Map<PrototypeHashCode, ColourHex>
|
||||
) : Map<PrototypeHashCode, ColourHex> by map {
|
||||
operator fun plus(other: TileColourDict): TileColourDict = TileColourDict(this.map + other.map)
|
||||
}
|
||||
|
||||
|
||||
/** Get the latest map-tile colours per server. */
|
||||
fun KStream<FactorioServerId, PrototypesUpdate>.createTilePrototypeTable()
|
||||
: KTable<FactorioServerId, TileColourDict> {
|
||||
|
||||
val pid = "createTilePrototypeTable"
|
||||
|
||||
return mapValues(
|
||||
"$pid.map-values"
|
||||
) { _, protoPacket: PrototypesUpdate ->
|
||||
val map = protoPacket
|
||||
.prototypes
|
||||
.filterIsInstance<FactorioPrototype.MapTile>()
|
||||
.associate { PrototypeHashCode(it) to it.mapColour.toHex() }
|
||||
|
||||
TileColourDict(map)
|
||||
}.filter("$pid.filterMapTileProtos") { _, dict: TileColourDict ->
|
||||
dict.isNotEmpty()
|
||||
}.peek { serverId, dict: TileColourDict ->
|
||||
println("$pid: server $serverId has TileColourDict[${dict.size}]: ${dict.entries.joinToString()}")
|
||||
}.repartition(
|
||||
repartitionedAs(
|
||||
"$pid.pre-table-repartition",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
// force, otherwise KTable-KTable FK join doesn't work
|
||||
numberOfPartitions = 1,
|
||||
)
|
||||
).toTable(
|
||||
"$pid.create-table",
|
||||
materializedAs(
|
||||
"$pid.output-store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@JvmInline
|
||||
@Serializable
|
||||
value class EntityPrototypeMap(
|
||||
private val map: Map<PrototypeHashCode, FactorioPrototype.Entity>
|
||||
) : Map<PrototypeHashCode, FactorioPrototype.Entity> by map {
|
||||
operator fun plus(other: EntityPrototypeMap): EntityPrototypeMap =
|
||||
EntityPrototypeMap(this.map + other.map)
|
||||
|
||||
operator fun get(protoId: PrototypeId): FactorioPrototype.Entity? =
|
||||
map[PrototypeHashCode(protoId)]
|
||||
|
||||
operator fun get(entity: FactorioEntityData): FactorioPrototype.Entity? =
|
||||
map[PrototypeHashCode(entity.protoId)]
|
||||
}
|
||||
|
||||
|
||||
/** Get the latest entity colours per server. */
|
||||
fun KStream<FactorioServerId, PrototypesUpdate>.createEntityPrototypeMap(
|
||||
callerPid: String
|
||||
): KTable<FactorioServerId, EntityPrototypeMap> {
|
||||
|
||||
val pid = "$callerPid.createEntityPrototypeMap"
|
||||
|
||||
return mapValues("$pid.colour-dict.build") { _, protoPacket: PrototypesUpdate ->
|
||||
val map = protoPacket
|
||||
.prototypes
|
||||
.filterIsInstance<FactorioPrototype.Entity>()
|
||||
.associateBy { entityProto ->
|
||||
PrototypeHashCode(entityProto)
|
||||
}
|
||||
|
||||
EntityPrototypeMap(map)
|
||||
}.filter("$pid.filterEntityProtos") { _, dict: EntityPrototypeMap ->
|
||||
dict.isNotEmpty()
|
||||
}.groupByKey(groupedAs("$pid.group-by-key"))
|
||||
.reduce(
|
||||
"$pid.merge",
|
||||
materializedWith(
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
)
|
||||
) { dict1: EntityPrototypeMap, dict2: EntityPrototypeMap ->
|
||||
dict1 + dict2
|
||||
}.toStream("$pid.")
|
||||
.filter("$pid.not-null.filter") { _, v -> v != null }
|
||||
.mapValues("$pid.not-null.require") { _, value ->
|
||||
requireNotNull(value)
|
||||
}.peek { serverId, dict: EntityPrototypeMap ->
|
||||
println("$pid: server $serverId has EntityPrototypeMap (size:${dict.size}): ${dict.entries.joinToString()}")
|
||||
}.repartition(
|
||||
repartitionedAs(
|
||||
"$pid.pre-table-repartition",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
// force, otherwise KTable-KTable FK join doesn't work
|
||||
numberOfPartitions = 1,
|
||||
)
|
||||
).toTable(
|
||||
"$pid.create-table",
|
||||
materializedAs(
|
||||
"$pid.output-store",
|
||||
kxsBinary.serde(),
|
||||
kxsBinary.serde(),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
|
@ -1,161 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import com.sksamuel.scrimage.ImmutableImage
|
||||
import com.sksamuel.scrimage.ScaleMethod
|
||||
import com.sksamuel.scrimage.nio.PngWriter
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
|
||||
import dev.adamko.kafkatorio.schema.packets.EventServerPacket
|
||||
import dev.adamko.kafkatorio.server.config.jsonMapper
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_BROADCAST_TO_WEBSOCKET
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.filter
|
||||
import dev.adamko.kotka.extensions.streams.map
|
||||
import dev.adamko.kotka.extensions.streams.mapValues
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import java.awt.image.BufferedImage
|
||||
import java.nio.file.Path
|
||||
import kotlin.collections.component1
|
||||
import kotlin.collections.component2
|
||||
import kotlin.io.path.fileSize
|
||||
import kotlin.math.abs
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.Topology
|
||||
import org.apache.kafka.streams.kstream.KStream
|
||||
|
||||
|
||||
private const val pid = "saveMapTiles"
|
||||
|
||||
private const val WEB_MAP_TILE_SIZE_PX = 256
|
||||
|
||||
|
||||
/**
|
||||
* @param[serverDataDir] [dev.adamko.kafkatorio.server.config.ApplicationProperties.serverDataDir]
|
||||
*/
|
||||
fun saveMapTiles(
|
||||
builder: StreamsBuilder,
|
||||
serverDataDir: Path,
|
||||
): Topology {
|
||||
|
||||
val subdividedMapChunkTilesDebounced: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
builder.stream(
|
||||
consumedAs(
|
||||
"$pid.consume.grouped-map-chunks",
|
||||
kxsBinary.serde(ServerMapChunkId.serializer()),
|
||||
kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
|
||||
),
|
||||
TOPIC_MAP_CHUNK_TERRAIN_COLOURED_STATE,
|
||||
TOPIC_MAP_CHUNK_RESOURCE_COLOURED_STATE,
|
||||
TOPIC_MAP_CHUNK_BUILDING_COLOURED_STATE,
|
||||
).filter("$pid.consume.grouped-map-chunks.filter-size") { _, y ->
|
||||
y.chunkId.chunkSize.lengthInTiles >= ChunkSize.CHUNK_256.lengthInTiles
|
||||
}
|
||||
|
||||
val savedChunkFilenames: KStream<ServerMapChunkId, ServerMapTilePngFilename> =
|
||||
subdividedMapChunkTilesDebounced.saveChunkAsImage(serverDataDir)
|
||||
|
||||
savedChunkFilenames.broadcastSavedTileUpdates()
|
||||
|
||||
return builder.build()
|
||||
}
|
||||
|
||||
|
||||
private fun KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>>.saveChunkAsImage(
|
||||
tileDirectory: Path,
|
||||
): KStream<ServerMapChunkId, ServerMapTilePngFilename> = mapValues(
|
||||
"$pid.save-tiles"
|
||||
) { chunkId: ServerMapChunkId, tileColours: ServerMapChunkTiles<ColourHex> ->
|
||||
|
||||
val chunkImage = ImmutableImage.filled(
|
||||
chunkId.chunkSize.lengthInTiles,
|
||||
chunkId.chunkSize.lengthInTiles,
|
||||
TRANSPARENT_AWT,
|
||||
BufferedImage.TYPE_INT_ARGB
|
||||
)
|
||||
|
||||
val chunkOriginX: Int = chunkId.chunkPosition.x * chunkId.chunkSize.lengthInTiles
|
||||
val chunkOriginY: Int = chunkId.chunkPosition.y * chunkId.chunkSize.lengthInTiles
|
||||
|
||||
val validPixelRangeX = 0 until chunkImage.width
|
||||
val validPixelRangeY = 0 until chunkImage.height
|
||||
|
||||
tileColours.map.forEach { (tilePosition, colour) ->
|
||||
val rgbColour = colour.toRgbColor()
|
||||
|
||||
val pixelX = abs(abs(tilePosition.x) - abs(chunkOriginX))
|
||||
val pixelY = abs(abs(tilePosition.y) - abs(chunkOriginY))
|
||||
|
||||
if (!(pixelX in validPixelRangeX && pixelY in validPixelRangeY)) {
|
||||
// TODO figure out why buildings on chunk boundaries aren't re-chunked, and so cause this error
|
||||
println(
|
||||
"ERROR - pixel XY:[$pixelX,$pixelY] was out of bounds (x:$validPixelRangeX, y:$validPixelRangeY). $tilePosition, $colour, chunkOriginXY[$chunkOriginX,$chunkOriginY], $chunkId"
|
||||
)
|
||||
} else {
|
||||
try {
|
||||
chunkImage.setColor(pixelX, pixelY, rgbColour)
|
||||
} catch (e: Exception) {
|
||||
println(
|
||||
"ERROR $e - pixel XY:[$pixelX,$pixelY] (x:$validPixelRangeX, y:$validPixelRangeY). $tilePosition, $colour, chunkOriginXY[$chunkOriginX,$chunkOriginY], $chunkId"
|
||||
)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val scaledImage =
|
||||
chunkImage.scaleTo(WEB_MAP_TILE_SIZE_PX, WEB_MAP_TILE_SIZE_PX, ScaleMethod.FastScale)
|
||||
|
||||
val filename = ServerMapTilePngFilename(chunkId)
|
||||
saveTile(filename, tileDirectory, scaledImage)
|
||||
filename
|
||||
}
|
||||
|
||||
|
||||
@Synchronized
|
||||
private fun saveTile(
|
||||
filename: ServerMapTilePngFilename,
|
||||
serverDataDir: Path,
|
||||
img: ImmutableImage,
|
||||
) {
|
||||
val chunkImageFile = serverDataDir.resolve(filename.value).toFile()
|
||||
if (chunkImageFile.parentFile.mkdirs()) {
|
||||
println("created new map tile parentFile directory ${chunkImageFile.absolutePath}")
|
||||
}
|
||||
val sizeBefore = chunkImageFile.takeIf { it.exists() }?.toPath()?.fileSize()
|
||||
|
||||
val savedTile = img.output(PngWriter.NoCompression, chunkImageFile)
|
||||
|
||||
val sizeAfter = savedTile.takeIf { it.exists() }?.toPath()?.fileSize()
|
||||
|
||||
val sizeDescription = if (sizeBefore != sizeAfter && sizeBefore != null) {
|
||||
"$sizeBefore/$sizeAfter"
|
||||
} else {
|
||||
"$sizeAfter"
|
||||
}
|
||||
println("savedTile $sizeDescription: $savedTile")
|
||||
}
|
||||
|
||||
|
||||
private fun KStream<ServerMapChunkId, ServerMapTilePngFilename>.broadcastSavedTileUpdates() {
|
||||
map<ServerMapChunkId, ServerMapTilePngFilename, FactorioServerId, EventServerPacket>(
|
||||
"$pid.broadcast.filename-update.convert",
|
||||
) { id, filename ->
|
||||
id.serverId to EventServerPacket.ChunkTileSaved(id, filename)
|
||||
}.to(
|
||||
TOPIC_BROADCAST_TO_WEBSOCKET,
|
||||
producedAs(
|
||||
"$pid.broadcast.filename-update.send",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(EventServerPacket.serializer()),
|
||||
)
|
||||
)
|
||||
}
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.jsonMapperKafkatorio
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.packets.EventServerPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
|
||||
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
|
||||
import dev.adamko.kafkatorio.schema.packets.SurfaceUpdate
|
||||
import dev.adamko.kafkatorio.server.processor.TOPIC_BROADCAST_TO_WEBSOCKET
|
||||
import dev.adamko.kafkatorio.server.processor.topicName
|
||||
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
import dev.adamko.kotka.extensions.consumedAs
|
||||
import dev.adamko.kotka.extensions.producedAs
|
||||
import dev.adamko.kotka.extensions.stream
|
||||
import dev.adamko.kotka.extensions.streams.forEach
|
||||
import dev.adamko.kotka.extensions.streams.map
|
||||
import dev.adamko.kotka.extensions.streams.to
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.serialization.encodeToString
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
|
||||
|
||||
private val jsonMapper: Json = Json {
|
||||
prettyPrint = false
|
||||
serializersModule = jsonMapperKafkatorio.serializersModule
|
||||
}
|
||||
|
||||
|
||||
private const val pid = "websocketBroadcast"
|
||||
|
||||
|
||||
fun broadcastToWebsocket(
|
||||
websocketServer: WebmapWebsocketServer,
|
||||
builder: StreamsBuilder,
|
||||
) {
|
||||
|
||||
builder.broadcastKafkatorioPackets()
|
||||
|
||||
builder.stream<FactorioServerId, EventServerPacket>(
|
||||
TOPIC_BROADCAST_TO_WEBSOCKET,
|
||||
consumedAs("$pid.packets-to-broadcast.consume", jsonMapper.serde(), jsonMapper.serde()),
|
||||
).forEach(
|
||||
"$pid.websocket-server.send-message"
|
||||
) { _, esPacket ->
|
||||
runBlocking {
|
||||
websocketServer.sendMessageToClients(jsonMapper.encodeToString(esPacket))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private fun StreamsBuilder.broadcastKafkatorioPackets() {
|
||||
val playerUpdatesStream = stream(
|
||||
consumedAs(
|
||||
"$pid.player-updates",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(KafkatorioPacket.serializer()),
|
||||
),
|
||||
PlayerUpdate::class.topicName,
|
||||
SurfaceUpdate::class.topicName,
|
||||
)
|
||||
|
||||
playerUpdatesStream.map<FactorioServerId, KafkatorioPacket, FactorioServerId, EventServerPacket>(
|
||||
"$pid.packet-updates.convert"
|
||||
) { serverId, packet ->
|
||||
serverId to EventServerPacket.Kafkatorio(packet)
|
||||
}.to(
|
||||
producedAs(
|
||||
"$pid.event-server-packets.produce",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(EventServerPacket.serializer()),
|
||||
)
|
||||
) { TOPIC_BROADCAST_TO_WEBSOCKET }
|
||||
}
|
||||
|
||||
|
||||
private fun StreamsBuilder.prepareServerUpdateBroadcast() {
|
||||
|
||||
val surfaceUpdatesStream = stream<FactorioServerId, KafkatorioPacket>(
|
||||
SurfaceUpdate::class.topicName,
|
||||
consumedAs("$pid.surface-updates", jsonMapper.serde(), jsonMapper.serde()),
|
||||
)
|
||||
|
||||
surfaceUpdatesStream.map<FactorioServerId, KafkatorioPacket, FactorioServerId, EventServerPacket>(
|
||||
"$pid.surface-updates.convert"
|
||||
) { serverId, serverUpdatePacket ->
|
||||
serverId to EventServerPacket.Kafkatorio(serverUpdatePacket)
|
||||
}.to(
|
||||
producedAs(
|
||||
"$pid.player-updates.stream",
|
||||
jsonMapper.serde(FactorioServerId.serializer()),
|
||||
jsonMapper.serde(EventServerPacket.serializer()),
|
||||
)
|
||||
) { TOPIC_BROADCAST_TO_WEBSOCKET }
|
||||
}
|
||||
|
|
@ -1,125 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.security
|
||||
|
||||
import com.auth0.jwt.JWT
|
||||
import com.auth0.jwt.algorithms.Algorithm
|
||||
import com.auth0.jwt.interfaces.DecodedJWT
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerToken
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import kotlin.reflect.jvm.jvmName
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.serialization.SerialName
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import okio.ByteString.Companion.decodeBase64
|
||||
|
||||
|
||||
class Authenticator(
|
||||
appProps: ApplicationProperties,
|
||||
) {
|
||||
private val algorithm: Algorithm = Algorithm.HMAC256(appProps.jwtSecret.value)
|
||||
|
||||
private val verifier = JWT.require(algorithm).build()
|
||||
|
||||
fun verifyJwt(token: String): FactorioServerToken? {
|
||||
return runCatching {
|
||||
val decodedJwt: DecodedJWT? = verifier.verify(token)
|
||||
|
||||
val payloadJson = decodedJwt?.payload?.decodeBase64()?.string(Charsets.UTF_8) ?: "{}"
|
||||
|
||||
jsonMapper.decodeFromString(
|
||||
KafkatorioJwtPayload.serializer(),
|
||||
payloadJson,
|
||||
).serverToken
|
||||
|
||||
}.onFailure { e ->
|
||||
if (e is CancellationException) throw e
|
||||
|
||||
println("[Authenticator] Could not decode JWT ${e::class.jvmName}: ${e.message}")
|
||||
}.getOrNull()
|
||||
}
|
||||
|
||||
companion object {
|
||||
private val jsonMapper = Json {
|
||||
ignoreUnknownKeys = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Serializable
|
||||
@SerialName("kafkatorio.auth.KafkatorioJwtPayload")
|
||||
private data class KafkatorioJwtPayload(
|
||||
@SerialName("sub")
|
||||
val serverToken: FactorioServerToken,
|
||||
/** unix epoch seconds*/
|
||||
@SerialName("iat")
|
||||
val issuedAtTime: Long,
|
||||
)
|
||||
|
||||
|
||||
//// https://github.com/auth0/java-jwt/blob/f799e589c3d43de318c7ca0f0c36e462297db973/lib/src/test/java/com/auth0/jwt/PemUtils.java
|
||||
//private object PemUtils {
|
||||
//
|
||||
// fun readPublicKey(filepath: String, algorithm: String): PublicKey {
|
||||
// val bytes = parsePemFile(File(filepath))
|
||||
// return getPublicKey(bytes, algorithm)
|
||||
// }
|
||||
//
|
||||
//// fun readPublicKey(reader: Reader, algorithm: String): PublicKey {
|
||||
//// return getPublicKey(bytes, algorithm)
|
||||
//// }
|
||||
//
|
||||
// fun readPrivateKey(filepath: String, algorithm: String): PrivateKey {
|
||||
// val bytes = parsePemFile(File(filepath))
|
||||
// return getPrivateKey(bytes, algorithm)
|
||||
// }
|
||||
//
|
||||
// private fun parsePemFile(pemFile: File): ByteArray {
|
||||
// return when {
|
||||
// !pemFile.isFile ->
|
||||
// throw FileNotFoundException("'${pemFile.absolutePath}' is not a file")
|
||||
//
|
||||
// !pemFile.exists() ->
|
||||
// throw FileNotFoundException("The file '${pemFile.absolutePath}' doesn't exist.")
|
||||
//
|
||||
// else ->
|
||||
// PemReader(FileReader(pemFile)).use { reader ->
|
||||
// reader.readPemObject().content
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// private fun getPublicKey(keyBytes: ByteArray, algorithm: String): PublicKey =
|
||||
// getKey(algorithm) {
|
||||
// val keySpec: EncodedKeySpec = X509EncodedKeySpec(keyBytes)
|
||||
// generatePublic(keySpec)
|
||||
// }
|
||||
//
|
||||
// private fun getPrivateKey(keyBytes: ByteArray, algorithm: String): PrivateKey =
|
||||
// getKey(algorithm) {
|
||||
// val keySpec: EncodedKeySpec = PKCS8EncodedKeySpec(keyBytes)
|
||||
// generatePrivate(keySpec)
|
||||
// }
|
||||
//
|
||||
//
|
||||
// private fun <K : Key> getKey(
|
||||
// algorithm: String,
|
||||
// generateKey: KeyFactory.() -> K
|
||||
// ): K = runCatching {
|
||||
// val kf = KeyFactory.getInstance(algorithm)
|
||||
// kf.generateKey()
|
||||
// }.getOrElse { e ->
|
||||
// when (e) {
|
||||
// is NoSuchAlgorithmException ->
|
||||
// throw IllegalStateException(
|
||||
// "Could not reconstruct key, algorithm '$algorithm' could not be found.",
|
||||
// e
|
||||
// )
|
||||
//
|
||||
// is InvalidKeySpecException ->
|
||||
// throw IllegalStateException("Could not reconstruct key", e)
|
||||
//
|
||||
// else -> throw e
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.socket
|
||||
|
||||
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
|
||||
|
||||
data class SyslogMsg(
|
||||
val src: Map<String, String?>,
|
||||
) {
|
||||
|
||||
val message: String?
|
||||
get() = src[SyslogFieldKeys.MESSAGE.field]
|
||||
|
||||
val headerAppName: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_APPNAME.field]
|
||||
|
||||
val headerHostName: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field]
|
||||
|
||||
val headerPri: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_PRI.field]
|
||||
|
||||
val headerPriSeverity: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field]
|
||||
|
||||
val headerPriFacility: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field]
|
||||
|
||||
val headerProcId: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_PROCID.field]
|
||||
|
||||
val headerTimestamp: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field]
|
||||
|
||||
val headerMsgId: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_MSGID.field]
|
||||
|
||||
val headerVersion: String?
|
||||
get() = src[SyslogFieldKeys.HEADER_VERSION.field]
|
||||
|
||||
val structuredBase: String?
|
||||
get() = src[SyslogFieldKeys.STRUCTURED_BASE.field]
|
||||
|
||||
val structuredElementIdFmt: String?
|
||||
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field]
|
||||
|
||||
val structuredElementIdPnameFmt: String?
|
||||
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field]
|
||||
|
||||
val structuredElementIdPnamePattern: String?
|
||||
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field]
|
||||
|
||||
}
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.socket
|
||||
|
||||
import com.github.palindromicity.syslog.AllowableDeviations
|
||||
import com.github.palindromicity.syslog.DefaultKeyProvider
|
||||
import com.github.palindromicity.syslog.Flat5424MessageHandler
|
||||
import com.github.palindromicity.syslog.NilPolicy
|
||||
import com.github.palindromicity.syslog.SyslogParser
|
||||
import com.github.palindromicity.syslog.SyslogParserBuilder
|
||||
import com.github.palindromicity.syslog.SyslogSpecification
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import io.ktor.network.selector.SelectorManager
|
||||
import io.ktor.network.sockets.Socket
|
||||
import io.ktor.network.sockets.aSocket
|
||||
import io.ktor.network.sockets.openReadChannel
|
||||
import io.ktor.utils.io.readUTF8Line
|
||||
import java.util.EnumSet
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.SharedFlow
|
||||
import kotlinx.coroutines.flow.asSharedFlow
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.yield
|
||||
|
||||
|
||||
class SyslogSocketServer(
|
||||
appProps: ApplicationProperties,
|
||||
) {
|
||||
|
||||
/** Incoming messages */
|
||||
private val serverSocketTcp = aSocket(SelectorManager(Dispatchers.IO))
|
||||
.tcp()
|
||||
.bind(
|
||||
appProps.socketServerHost,
|
||||
appProps.socketServerPort,
|
||||
)
|
||||
|
||||
|
||||
private val syslogParser: SyslogParser<Map<String, String>> =
|
||||
SyslogParserBuilder<Map<String, String>>()
|
||||
.forSpecification(SyslogSpecification.RFC_5424)
|
||||
.withSyslogBuilder(
|
||||
Flat5424MessageHandler(
|
||||
DefaultKeyProvider(),
|
||||
NilPolicy.OMIT,
|
||||
EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION),
|
||||
)
|
||||
).build()
|
||||
|
||||
|
||||
private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
|
||||
val messages: SharedFlow<SyslogMsg>
|
||||
get() = _messages.asSharedFlow()
|
||||
|
||||
|
||||
suspend fun start(): Unit = coroutineScope {
|
||||
log("started listening: ${serverSocketTcp.localAddress}")
|
||||
|
||||
while (isActive) {
|
||||
val socket: Socket = serverSocketTcp.accept()
|
||||
log("new socket ${socket.description()}")
|
||||
|
||||
launch {
|
||||
val channel = socket.openReadChannel()
|
||||
try {
|
||||
while (socket.socketContext.isActive && !channel.isClosedForRead) {
|
||||
val line = runCatching {
|
||||
channel.readUTF8Line()
|
||||
}.fold(
|
||||
onSuccess = { line ->
|
||||
if (line == null) {
|
||||
log("received 'null' from ${socket.description()} - closing connection")
|
||||
socket.close()
|
||||
}
|
||||
// log("received '$line' from ${socket.description()}")
|
||||
line
|
||||
},
|
||||
onFailure = { ex ->
|
||||
if (ex is CancellationException) throw ex
|
||||
|
||||
log("reading line from socket failed ${socket.description()}, cause: $ex")
|
||||
null
|
||||
},
|
||||
)
|
||||
if (line != null) {
|
||||
val syslog = SyslogMsg(syslogParser.parseLine(line))
|
||||
_messages.emit(syslog)
|
||||
}
|
||||
yield()
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
if (e is CancellationException) throw e
|
||||
log("closing socket ${socket.description()}, cause: ${e::class.java.name} ${e.message}")
|
||||
e.printStackTrace()
|
||||
withContext(Dispatchers.IO) { socket.close() }
|
||||
}
|
||||
}
|
||||
|
||||
yield()
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
private fun log(msg: String) = println("[SyslogSocketServer] $msg")
|
||||
|
||||
private fun Socket.description(): String = "Socket[local:$localAddress, remote:$remoteAddress]"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.web.rest
|
||||
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.PlayerIndex
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTilePngFilename
|
||||
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.config.jsonMapper
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.http.content.EntityTagVersion
|
||||
import io.ktor.http.content.versions
|
||||
import io.ktor.resources.Resource
|
||||
import io.ktor.server.application.ApplicationCall
|
||||
import io.ktor.server.application.call
|
||||
import io.ktor.server.http.content.LastModifiedVersion
|
||||
import io.ktor.server.resources.get
|
||||
import io.ktor.server.response.respond
|
||||
import io.ktor.server.response.respondFile
|
||||
import io.ktor.server.routing.Routing
|
||||
import io.ktor.util.combineSafe
|
||||
import io.ktor.util.pipeline.PipelineContext
|
||||
import kotlin.io.path.Path
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.encodeToJsonElement
|
||||
|
||||
|
||||
fun Routing.factorioServersRoutes(
|
||||
appProps: ApplicationProperties,
|
||||
) {
|
||||
|
||||
get<FactorioServers> {
|
||||
val serverIds = appProps.kafkatorioServers.values.toSet().sortedBy { it.id }
|
||||
val serverIdsJson = jsonMapper.encodeToJsonElement(serverIds)
|
||||
call.respond(serverIdsJson)
|
||||
}
|
||||
|
||||
get<FactorioServer.MapData.Tile> { tile ->
|
||||
handleTileRequest(tile, appProps)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private suspend fun PipelineContext<Unit, ApplicationCall>.handleTileRequest(
|
||||
tile: FactorioServer.MapData.Tile,
|
||||
appProps: ApplicationProperties,
|
||||
) {
|
||||
|
||||
val tileFilename = ServerMapTilePngFilename(
|
||||
tile.serverId,
|
||||
tile.layer,
|
||||
tile.surfaceIndex,
|
||||
tile.zoomLevel,
|
||||
tile.chunkX,
|
||||
tile.chunkY,
|
||||
)
|
||||
|
||||
val tileFile = appProps.serverDataDir.combineSafe(Path(tileFilename.value))
|
||||
|
||||
if (tileFile.exists()) {
|
||||
call.respondFile(tileFile) {
|
||||
versions = versions + listOf(
|
||||
EntityTagVersion(tileFile.lastModified().hashCode().toString()),
|
||||
LastModifiedVersion(tileFile.lastModified()),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
// It's not a client or server error to request a non-existent tile, it just doesn't exist
|
||||
// (yet). So return 304.
|
||||
call.respond(HttpStatusCode.NotModified)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Serializable
|
||||
@Resource("kafkatorio/data/servers")
|
||||
private class FactorioServers
|
||||
|
||||
|
||||
@Serializable
|
||||
@Resource("kafkatorio/data/servers/{factorioServerId}")
|
||||
data class FactorioServer(
|
||||
val factorioServerId: FactorioServerId,
|
||||
) {
|
||||
|
||||
@Serializable
|
||||
@Resource("map")
|
||||
data class MapData(
|
||||
val server: FactorioServer,
|
||||
) {
|
||||
|
||||
@Serializable
|
||||
@Resource("layers/{layer}/s{surfaceIndex}/z{zoomLevel}/x{chunkX}/y{chunkY}.png")
|
||||
data class Tile(
|
||||
val mapData: MapData,
|
||||
|
||||
val layer: ServerMapTileLayer,
|
||||
val surfaceIndex: SurfaceIndex,
|
||||
val zoomLevel: Int,
|
||||
val chunkX: Int,
|
||||
val chunkY: Int,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
@Resource("buildings/s{surfaceIndex}")
|
||||
data class Buildings(
|
||||
val mapData: MapData,
|
||||
val surfaceIndex: SurfaceIndex,
|
||||
// a list of buildings... (Sealed interface, different types?)
|
||||
)
|
||||
}
|
||||
|
||||
@Serializable
|
||||
@Resource("players")
|
||||
data class Players(
|
||||
val server: FactorioServer,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
@Resource("{playerId}")
|
||||
data class Player(
|
||||
val server: FactorioServer,
|
||||
val playerIndex: PlayerIndex,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
private val FactorioServer.MapData.Tile.serverId: FactorioServerId
|
||||
get() = mapData.server.factorioServerId
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.web.rest
|
||||
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import io.ktor.http.ContentType
|
||||
import io.ktor.http.HttpStatusCode
|
||||
import io.ktor.http.isSuccess
|
||||
import io.ktor.serialization.kotlinx.json.json
|
||||
import io.ktor.server.application.Application
|
||||
import io.ktor.server.application.install
|
||||
import io.ktor.server.application.log
|
||||
import io.ktor.server.plugins.autohead.AutoHeadResponse
|
||||
import io.ktor.server.plugins.callloging.CallLogging
|
||||
import io.ktor.server.plugins.compression.Compression
|
||||
import io.ktor.server.plugins.compression.excludeContentType
|
||||
import io.ktor.server.plugins.compression.minimumSize
|
||||
import io.ktor.server.plugins.conditionalheaders.ConditionalHeaders
|
||||
import io.ktor.server.plugins.contentnegotiation.ContentNegotiation
|
||||
import io.ktor.server.plugins.partialcontent.PartialContent
|
||||
import io.ktor.server.plugins.statuspages.StatusPages
|
||||
import io.ktor.server.resources.Resources
|
||||
import io.ktor.server.response.respondText
|
||||
import io.ktor.server.routing.routing
|
||||
|
||||
|
||||
fun Application.webmapTileServer(appProps: ApplicationProperties) {
|
||||
|
||||
install(AutoHeadResponse)
|
||||
install(CallLogging) {
|
||||
filter { call ->
|
||||
// only log failures
|
||||
!(call.response.status()?.isSuccess() ?: false)
|
||||
}
|
||||
}
|
||||
install(Compression) {
|
||||
default()
|
||||
minimumSize(1024)
|
||||
excludeContentType(ContentType.Video.Any)
|
||||
}
|
||||
install(ConditionalHeaders)
|
||||
install(ContentNegotiation) {
|
||||
json()
|
||||
}
|
||||
install(PartialContent)
|
||||
install(Resources)
|
||||
|
||||
install(StatusPages) {
|
||||
exception<Throwable> { call, cause ->
|
||||
this@webmapTileServer.log.error("$call caused exception - returning 500", cause)
|
||||
call.respondText(text = "500: $cause", status = HttpStatusCode.InternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
routing {
|
||||
factorioServersRoutes(appProps)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.web
|
||||
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.web.rest.webmapTileServer
|
||||
import dev.adamko.kafkatorio.server.web.websocket.WebmapWebsocketServer
|
||||
import dev.adamko.kafkatorio.server.web.websocket.webmapWebsocketServer
|
||||
import io.ktor.server.cio.CIO
|
||||
import io.ktor.server.engine.embeddedServer
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
|
||||
|
||||
suspend fun startWebServer(
|
||||
appProps: ApplicationProperties,
|
||||
wsServer: WebmapWebsocketServer,
|
||||
): Unit = coroutineScope {
|
||||
|
||||
embeddedServer(
|
||||
factory = CIO,
|
||||
port = appProps.webPort.value,
|
||||
) {
|
||||
|
||||
webmapWebsocketServer(wsServer)
|
||||
|
||||
webmapTileServer(appProps)
|
||||
|
||||
}.start(wait = true)
|
||||
}
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.web.websocket
|
||||
|
||||
import io.ktor.websocket.CloseReason
|
||||
import io.ktor.websocket.DefaultWebSocketSession
|
||||
import io.ktor.websocket.Frame
|
||||
import io.ktor.websocket.send
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import kotlin.time.TimeSource
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.channels.BufferOverflow
|
||||
import kotlinx.coroutines.coroutineScope
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.flow.MutableSharedFlow
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.launchIn
|
||||
import kotlinx.coroutines.flow.onEach
|
||||
import kotlinx.coroutines.flow.receiveAsFlow
|
||||
import kotlinx.coroutines.flow.update
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.job
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.yield
|
||||
|
||||
|
||||
class WebmapWebsocketServer {
|
||||
|
||||
private val startTime = TimeSource.Monotonic.markNow()
|
||||
|
||||
private val clients: MutableStateFlow<List<WSConnection>> = MutableStateFlow(emptyList())
|
||||
|
||||
private val messagesToClients: MutableSharedFlow<String> = MutableSharedFlow(
|
||||
extraBufferCapacity = Int.MAX_VALUE,
|
||||
onBufferOverflow = BufferOverflow.DROP_OLDEST
|
||||
)
|
||||
|
||||
private val messagesFromClients: MutableSharedFlow<Frame> = MutableSharedFlow(
|
||||
extraBufferCapacity = Int.MAX_VALUE,
|
||||
onBufferOverflow = BufferOverflow.DROP_OLDEST
|
||||
)
|
||||
|
||||
|
||||
suspend fun bindClient(session: DefaultWebSocketSession): Unit = coroutineScope {
|
||||
val client = WSConnection(session)
|
||||
|
||||
clients.update { clients -> clients + client }
|
||||
|
||||
client.coroutineContext.job.invokeOnCompletion { e ->
|
||||
log("client ${client.name} completed ${e ?: "successfully"}")
|
||||
clients.update { clients ->
|
||||
(clients - client).filter { it.isActive }
|
||||
}
|
||||
}
|
||||
|
||||
messagesToClients.onEach { msg ->
|
||||
runCatching {
|
||||
if (client.isActive) {
|
||||
client.outgoing.send(Frame.Text(msg))
|
||||
}
|
||||
}.onFailure { e ->
|
||||
if (e is CancellationException) throw e
|
||||
}
|
||||
}.launchIn(client)
|
||||
|
||||
client.launch {
|
||||
client.incoming.receiveAsFlow().collect(messagesFromClients)
|
||||
}
|
||||
|
||||
client.launch {
|
||||
while (isActive) {
|
||||
runCatching {
|
||||
client.send("elapsed ${startTime.elapsedNow()}")
|
||||
}.onFailure { e ->
|
||||
if (e is CancellationException) throw e
|
||||
}
|
||||
delay(10.seconds)
|
||||
yield()
|
||||
}
|
||||
}
|
||||
|
||||
log("client ${client.name} is up and running...")
|
||||
val closedReason: CloseReason? = client.closeReason.await()
|
||||
|
||||
log("client ${client.name} closed ${closedReason ?: "successfully"}")
|
||||
|
||||
coroutineContext.cancel()
|
||||
}
|
||||
|
||||
|
||||
suspend fun sendMessageToClients(msg: String) {
|
||||
log("sending message to clients: $msg")
|
||||
messagesToClients.emit(msg)
|
||||
}
|
||||
|
||||
|
||||
private data class WSConnection(
|
||||
private val session: DefaultWebSocketSession,
|
||||
) : DefaultWebSocketSession by session {
|
||||
val name: String = "user${lastId.getAndIncrement()}"
|
||||
|
||||
override suspend fun send(frame: Frame): Unit = runCatching {
|
||||
super.send(frame)
|
||||
}.fold(
|
||||
onSuccess = {
|
||||
// log("sent message to client $name: $frame")
|
||||
},
|
||||
onFailure = { e ->
|
||||
if (e is CancellationException) throw e
|
||||
|
||||
log("failed to send message to client $name: $e")
|
||||
}
|
||||
)
|
||||
|
||||
override fun equals(other: Any?): Boolean = name == other
|
||||
override fun hashCode(): Int = name.hashCode()
|
||||
|
||||
companion object {
|
||||
private val lastId = AtomicInteger(0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
companion object {
|
||||
|
||||
private fun log(msg: String) = println("[WebmapWebsocketServer] $msg")
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.web.websocket
|
||||
|
||||
import io.ktor.server.application.Application
|
||||
import io.ktor.server.application.install
|
||||
import io.ktor.server.routing.routing
|
||||
import io.ktor.server.websocket.WebSockets
|
||||
import io.ktor.server.websocket.webSocket
|
||||
import io.ktor.websocket.WebSocketDeflateExtension
|
||||
import java.util.zip.Deflater
|
||||
|
||||
|
||||
fun Application.webmapWebsocketServer(
|
||||
wsServer: WebmapWebsocketServer,
|
||||
) {
|
||||
install(WebSockets) {
|
||||
extensions {
|
||||
install(WebSocketDeflateExtension) {
|
||||
// Compression level to use for java.util.zip.Deflater
|
||||
compressionLevel = Deflater.DEFAULT_COMPRESSION
|
||||
|
||||
// Prevent compressing small outgoing frames
|
||||
compressIfBiggerThan(bytes = 4 * 1024)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
routing {
|
||||
// TODO only send a ws client message for a single Factorio-sever. Make the URL server specific.
|
||||
// /kafkatorio/data/servers/{serverId}/ws
|
||||
webSocket("kafkatorio/ws") {
|
||||
wsServer.bindClient(this)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
# Factorio Events Processor config
|
||||
|
||||
web-port: ${SERVER_PORT:-8080}
|
||||
server-data-dir: ${SERVER_DATA_DIR:-./server-data}
|
||||
|
||||
|
||||
socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}
|
||||
#socket-server-port: ${SOCKET_SERVER_PORT:-12092}
|
||||
|
||||
|
||||
kafka-admin-config:
|
||||
client.id: kafkatorio.events-server-admin
|
||||
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}
|
||||
|
||||
|
||||
kafka-streams-config:
|
||||
application.id: kafkatorio-events-processor
|
||||
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}
|
||||
state.dir: ${KAFKA_STREAMS_STATE_DIR:-./kafka-state}
|
||||
num.stream.threads: 1
|
||||
processing.guarantee: exactly_once_v2
|
||||
auto.offset.reset: earliest
|
||||
default.key.serde: org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
default.value.serde: org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
|
||||
# default.deserialization.exception.handler: org.apache.kafka.streams.errors.LogAndContinueExceptionHandler
|
||||
|
||||
#buffered.records.per.partition: 0
|
||||
#poll.ms: 10000
|
||||
|
||||
# 31457280 : 30MB
|
||||
max.request.size: 31457280
|
||||
buffer.memory: 31457280
|
||||
|
||||
compression.type: snappy
|
||||
# linger.ms=5
|
||||
|
||||
# fetch.message.max.bytes=31457280
|
||||
# 100mb 104857600
|
||||
|
||||
|
||||
#kafka:
|
||||
# bootstrap-servers:
|
||||
# - "http://localhost:9092"
|
||||
# config: |
|
||||
# application.id=kafkatorio-events-processor
|
||||
# bootstrap.servers={{bootstrap-servers}}
|
||||
# state.dir=./kafka-state
|
||||
# num.stream.threads=1
|
||||
# processing.guarantee=exactly_once_v2
|
||||
# auto.offset.reset=earliest
|
||||
# default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
# default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde
|
||||
# default.deserialization.exception.handler=org.apache.kafka.streams.errors.LogAndContinueExceptionHandler
|
||||
# #buffered.records.per.partition=0
|
||||
# #poll.ms=10000
|
||||
#
|
||||
# # 31457280 = 30MB
|
||||
# max.request.size=31457280
|
||||
# buffer.memory=31457280
|
||||
#
|
||||
# compression.type=snappy
|
||||
# # linger.ms=5
|
||||
#
|
||||
# # fetch.message.max.bytes=31457280
|
||||
# # 100mb 104857600
|
||||
#
|
||||
#tileServer:
|
||||
# webmapTileDir: 'D:\Users\Adam\Projects\games\kafkatorio\modules\events-server\kafkatorio-tiles'
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
<root>
|
||||
<item name='org.apache.kafka.streams.state.ReadOnlyKeyValueStore V get(K)'>
|
||||
<annotation name='org.jetbrains.annotations.Nullable'/>
|
||||
</item>
|
||||
</root>
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
<configuration>
|
||||
|
||||
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<withJansi>false</withJansi>
|
||||
<!-- encoders are assigned the type
|
||||
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
|
||||
<encoder>
|
||||
<pattern>%cyan(%d{HH:mm:ss.SSS}) %white([%thread]) %highlight(%-5level) %magenta(%logger{36}) - %msg%n
|
||||
</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="org.apache.kafka" level="info" additivity="false">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</logger>
|
||||
|
||||
<root level="info">
|
||||
<appender-ref ref="STDOUT"/>
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
package dev.adamko.kafkatorio.processor.misc
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
|
||||
import dev.adamko.kafkatorio.server.processor.topology.ServerMapChunkTiles
|
||||
import java.util.UUID
|
||||
import kotlinx.serialization.encodeToByteArray
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
|
||||
class Blah {
|
||||
|
||||
@Test
|
||||
fun asdad() {
|
||||
|
||||
val tiles = ServerMapChunkTiles<ColourHex>(
|
||||
chunkId = ServerMapChunkId(
|
||||
serverId = FactorioServerId(UUID.randomUUID().toString()),
|
||||
layer = ServerMapTileLayer.BUILDING,
|
||||
chunkPosition = MapChunkPosition(Int.MAX_VALUE, Int.MAX_VALUE, ChunkSize.MAX),
|
||||
surfaceIndex = SurfaceIndex(UInt.MAX_VALUE),
|
||||
),
|
||||
buildMap {
|
||||
repeat(512) { x ->
|
||||
repeat(512) { y ->
|
||||
put(MapTilePosition(x, y), ColourHex(255u, 255u, 255u, 255u))
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
val encode = kxsBinary.encodeToByteArray(tiles)
|
||||
|
||||
encode.inputStream()
|
||||
val size = encode.size * Byte.SIZE_BYTES
|
||||
|
||||
println(size)
|
||||
|
||||
|
||||
val map2 = buildList {
|
||||
repeat(512) { x ->
|
||||
repeat(512) { y ->
|
||||
val element = (x to y) to Int.MAX_VALUE
|
||||
add(element)
|
||||
}
|
||||
}
|
||||
}
|
||||
// .toTypedArray()
|
||||
val encode2 = kxsBinary.encodeToByteArray(map2)
|
||||
|
||||
encode2.inputStream()
|
||||
val size2 = encode2.size * Byte.SIZE_BYTES
|
||||
|
||||
println(size2)
|
||||
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,48 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.toJavaDuration
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.TestInputTopic
|
||||
import org.apache.kafka.streams.TopologyTestDriver
|
||||
|
||||
class FactorioServerPacketsTest : FunSpec({
|
||||
|
||||
|
||||
})
|
||||
|
||||
private class Scenario(
|
||||
streamsBuilder: StreamsBuilder = StreamsBuilder(),
|
||||
|
||||
groupedMapChunksInputTopicName: String = "kafkatorio.state.map-chunk.terrain.colour",
|
||||
|
||||
val serverId: FactorioServerId = FactorioServerId("test-server-id"),
|
||||
) : AutoCloseable {
|
||||
|
||||
init {
|
||||
factorioServerPacketStream(streamsBuilder)
|
||||
}
|
||||
|
||||
private val testDriver: TopologyTestDriver = TopologyTestDriver(streamsBuilder.build())
|
||||
|
||||
val groupedMapChunksInputTopic: TestInputTopic<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
testDriver.createInputTopic(
|
||||
groupedMapChunksInputTopicName,
|
||||
kxsBinary.serde<ServerMapChunkId>().serializer(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>().serializer(),
|
||||
)
|
||||
|
||||
operator fun plusAssign(wallClockTime: Duration) {
|
||||
testDriver.advanceWallClockTime(wallClockTime.toJavaDuration())
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
testDriver.close()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,185 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.processor.topology
|
||||
|
||||
import dev.adamko.kafkatorio.library.iterateTiles
|
||||
import dev.adamko.kafkatorio.library.kxsBinary
|
||||
import dev.adamko.kafkatorio.library.toMapChunkPosition
|
||||
import dev.adamko.kafkatorio.schema.common.ChunkSize
|
||||
import dev.adamko.kafkatorio.schema.common.ColourHex
|
||||
import dev.adamko.kafkatorio.schema.common.FactorioServerId
|
||||
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
|
||||
import dev.adamko.kafkatorio.schema.common.MapTilePosition
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapChunkId
|
||||
import dev.adamko.kafkatorio.schema.common.ServerMapTileLayer
|
||||
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
|
||||
import dev.adamko.kotka.kxs.serde
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import io.kotest.property.Arb
|
||||
import io.kotest.property.PropTestConfig
|
||||
import io.kotest.property.RandomSource
|
||||
import io.kotest.property.arbitrary.arbitrary
|
||||
import io.kotest.property.arbitrary.int
|
||||
import io.kotest.property.arbitrary.uByte
|
||||
import io.kotest.property.checkAll
|
||||
import java.nio.file.Path
|
||||
import java.time.LocalTime
|
||||
import java.time.format.DateTimeFormatter
|
||||
import kotlin.io.path.Path
|
||||
import kotlin.io.path.absolutePathString
|
||||
import kotlin.time.Duration
|
||||
import kotlin.time.Duration.Companion.minutes
|
||||
import kotlin.time.Duration.Companion.seconds
|
||||
import kotlin.time.toJavaDuration
|
||||
import kotlinx.coroutines.delay
|
||||
import org.apache.kafka.streams.StreamsBuilder
|
||||
import org.apache.kafka.streams.TestInputTopic
|
||||
import org.apache.kafka.streams.TopologyTestDriver
|
||||
|
||||
|
||||
class SaveMapChunksTest : FunSpec({
|
||||
|
||||
test("save tile") {
|
||||
|
||||
Scenario().use { scenario ->
|
||||
|
||||
val chunkTilesArb = serverMapChunkTilesArb(scenario.serverId)
|
||||
|
||||
checkAll(
|
||||
config = PropTestConfig(iterations = 1),
|
||||
chunkTilesArb,
|
||||
) { chunkTiles ->
|
||||
|
||||
scenario.groupedMapChunksInputTopic.pipeInput(
|
||||
chunkTiles.chunkId,
|
||||
chunkTiles,
|
||||
)
|
||||
|
||||
scenario += 1.minutes
|
||||
delay(5.seconds)
|
||||
scenario += 1.minutes
|
||||
// delay(5.seconds)
|
||||
scenario += 1.minutes
|
||||
// delay(5.seconds)
|
||||
scenario += 1.minutes
|
||||
// delay(5.seconds)
|
||||
|
||||
println("file:///${scenario.outputDir.absolutePathString()}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}) {
|
||||
|
||||
class Scenario(
|
||||
streamsBuilder: StreamsBuilder = StreamsBuilder(),
|
||||
|
||||
groupedMapChunksInputTopicName: String = "kafkatorio.state.map-chunk.terrain.colour",
|
||||
|
||||
val serverId: FactorioServerId = FactorioServerId("test-server-id"),
|
||||
) : AutoCloseable {
|
||||
|
||||
val outputDir: Path = run {
|
||||
val now = LocalTime.now().format(DateTimeFormatter.ISO_TIME).filter { it.isLetterOrDigit() }
|
||||
Path("build/test/save-map-chunks-${now}")
|
||||
}
|
||||
private val topology = saveMapTiles(streamsBuilder, outputDir)
|
||||
private val testDriver: TopologyTestDriver = TopologyTestDriver(topology)
|
||||
|
||||
val groupedMapChunksInputTopic: TestInputTopic<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
|
||||
testDriver.createInputTopic(
|
||||
groupedMapChunksInputTopicName,
|
||||
kxsBinary.serde<ServerMapChunkId>().serializer(),
|
||||
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>().serializer(),
|
||||
)
|
||||
|
||||
operator fun plusAssign(wallClockTime: Duration) {
|
||||
testDriver.advanceWallClockTime(wallClockTime.toJavaDuration())
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
testDriver.close()
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
|
||||
private fun mapChunkPositionArb(
|
||||
xArb: Arb<Int> = Arb.int(-10..10),
|
||||
yArb: Arb<Int> = Arb.int(-10..10),
|
||||
chunkSize: ChunkSize = ChunkSize.MAX,
|
||||
): Arb<MapChunkPosition> = arbitrary {
|
||||
MapChunkPosition(xArb.bind(), yArb.bind(), chunkSize)
|
||||
}
|
||||
|
||||
private fun colourHexArb(
|
||||
redArb: Arb<UByte> = Arb.uByte(),
|
||||
greenArb: Arb<UByte> = Arb.uByte(),
|
||||
blueArb: Arb<UByte> = Arb.uByte(),
|
||||
alphaArb: Arb<UByte> = Arb.uByte(),
|
||||
): Arb<ColourHex> = arbitrary {
|
||||
ColourHex(
|
||||
redArb.bind(),
|
||||
greenArb.bind(),
|
||||
blueArb.bind(),
|
||||
alphaArb.bind(),
|
||||
)
|
||||
}
|
||||
|
||||
private fun serverMapChunkIdArb(
|
||||
serverId: FactorioServerId,
|
||||
chunkSize: ChunkSize = ChunkSize.MAX,
|
||||
chunkPosition: Arb<MapChunkPosition> = mapChunkPositionArb(chunkSize = chunkSize),
|
||||
surfaceIndex: SurfaceIndex = SurfaceIndex(1u),
|
||||
): Arb<ServerMapChunkId> = arbitrary {
|
||||
ServerMapChunkId(
|
||||
serverId = serverId,
|
||||
layer = ServerMapTileLayer.RESOURCE,
|
||||
chunkPosition = chunkPosition.bind(),
|
||||
surfaceIndex = surfaceIndex,
|
||||
)
|
||||
}
|
||||
|
||||
private fun tilesInChunkArb(
|
||||
chunkId: ServerMapChunkId
|
||||
): Arb<Map<MapTilePosition, ColourHex>> = arbitrary {
|
||||
|
||||
val tileColours = mutableMapOf<MapChunkPosition, ColourHex>()
|
||||
|
||||
println("chunk size: ${chunkId.chunkSize} ${chunkId.chunkSize.lengthInTiles}")
|
||||
|
||||
val map = chunkId.chunkPosition
|
||||
.iterateTiles()
|
||||
.asSequence()
|
||||
.map { tilePos ->
|
||||
val colour =
|
||||
tileColours.getOrPut(tilePos.toMapChunkPosition(ChunkSize.MIN)) {
|
||||
colourHexArb().sample(RandomSource.default()).value
|
||||
}
|
||||
tilePos to colour
|
||||
}
|
||||
// .take(100)
|
||||
.toMap()
|
||||
|
||||
println(map.size)
|
||||
|
||||
map
|
||||
}
|
||||
|
||||
fun serverMapChunkTilesArb(
|
||||
serverId: FactorioServerId,
|
||||
): Arb<ServerMapChunkTiles<ColourHex>> {
|
||||
|
||||
val serverMapChunkId = serverMapChunkIdArb(serverId)
|
||||
|
||||
return arbitrary {
|
||||
val chunkId = serverMapChunkId.bind()
|
||||
|
||||
val tileColoursArb = tilesInChunkArb(chunkId)
|
||||
|
||||
ServerMapChunkTiles(
|
||||
chunkId = chunkId,
|
||||
map = tileColoursArb.bind()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
package dev.adamko.kafkatorio.server.security
|
||||
|
||||
import dev.adamko.kafkatorio.schema.common.MaskedValue
|
||||
import dev.adamko.kafkatorio.server.config.ApplicationProperties
|
||||
import dev.adamko.kafkatorio.server.config.PortVal
|
||||
import io.kotest.core.spec.style.FunSpec
|
||||
import io.kotest.matchers.nulls.shouldBeNull
|
||||
import io.kotest.matchers.nulls.shouldNotBeNull
|
||||
import kotlin.io.path.Path
|
||||
|
||||
|
||||
class AuthenticatorTest : FunSpec({
|
||||
|
||||
test("verify correct JWT is verified") {
|
||||
|
||||
val authenticator = Authenticator(
|
||||
testApplicationProperties(
|
||||
jwtSecret = "123 456 789"
|
||||
)
|
||||
)
|
||||
|
||||
val result = authenticator.verifyJwt(jwtMessage)
|
||||
|
||||
result.shouldNotBeNull()
|
||||
}
|
||||
|
||||
test("verify invalid JWT returns null") {
|
||||
|
||||
val authenticator = Authenticator(
|
||||
testApplicationProperties(
|
||||
jwtSecret = "wrong"
|
||||
)
|
||||
)
|
||||
|
||||
val result = authenticator.verifyJwt(jwtMessage)
|
||||
|
||||
result.shouldBeNull()
|
||||
}
|
||||
|
||||
}) {
|
||||
|
||||
companion object {
|
||||
/** Key: `123 465 789` */
|
||||
private val jwtMessage = """
|
||||
eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.ToFLkPBs_EKF4B2bRQyVPfkKrB-x6WgyAq6KsR9oHwg
|
||||
""".trimIndent()
|
||||
|
||||
private fun testApplicationProperties(
|
||||
jwtSecret: String,
|
||||
) = ApplicationProperties(
|
||||
webPort = PortVal.DEFAULT,
|
||||
serverDataDir = Path(""),
|
||||
kafkaAdminConfig = emptyMap(),
|
||||
kafkaStreamsConfig = emptyMap(),
|
||||
socketServerHost = "",
|
||||
socketServerPort = 9999,
|
||||
jwtSecret = MaskedValue(jwtSecret),
|
||||
kafkatorioServers = emptyMap(),
|
||||
)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue