move from schema2 to schema

This commit is contained in:
Adam 2022-04-20 17:18:09 +02:00
parent 6bfcd30558
commit b0a6151c36
30 changed files with 256 additions and 318 deletions

View file

@ -1,14 +1,14 @@
package dev.adamko.kafkatorio.processor.admin
import dev.adamko.kafkatorio.schema2.ConfigurationUpdate
import dev.adamko.kafkatorio.schema2.ConsoleChatUpdate
import dev.adamko.kafkatorio.schema2.ConsoleCommandUpdate
import dev.adamko.kafkatorio.schema2.EntityUpdate
import dev.adamko.kafkatorio.schema2.KafkatorioPacketData2
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.PlayerUpdate
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kafkatorio.schema2.SurfaceUpdate
import dev.adamko.kafkatorio.schema.packets.ConfigurationUpdate
import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate
import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate
import dev.adamko.kafkatorio.schema.packets.EntityUpdate
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kafkatorio.schema.packets.SurfaceUpdate
import kotlin.reflect.KClass
@ -16,7 +16,7 @@ private const val DOMAIN = "kafkatorio"
const val TOPIC_SRC_SERVER_LOG = "$DOMAIN.src.server-log"
// "factorio-server-log"
const val TOPIC_GROUPED_MAP_CHUNKS_STATE = "$DOMAIN.state.map-chunks.grouped"
@ -27,12 +27,7 @@ fun allTopics(): Set<String> = buildSet {
addAll(packetTopicNames.values)
}
//fun KafkatorioPacket2.topicName(): String = data.topicName()
//fun KafkatorioPacketData2.topicName(): String = "$DOMAIN.packet.${topicName}"
val KafkatorioPacketData2.topicName: String
val KafkatorioPacketData.topicName: String
get() = when (this) {
is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class)
is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class)
@ -45,7 +40,7 @@ val KafkatorioPacketData2.topicName: String
}
private val packetTopicNames: Map<KClass<out KafkatorioPacketData2>, String> = mapOf(
private val packetTopicNames: Map<KClass<out KafkatorioPacketData>, String> = mapOf(
ConfigurationUpdate::class to "configuration",
ConsoleChatUpdate::class to "console-chat",
ConsoleCommandUpdate::class to "console-command",
@ -58,47 +53,5 @@ private val packetTopicNames: Map<KClass<out KafkatorioPacketData2>, String> = m
"$DOMAIN.packet.$v"
}
val KClass<out KafkatorioPacketData2>.topicName: String
val KClass<out KafkatorioPacketData>.topicName: String
get() = packetTopicNames.getValue(this)
//fun packetTopic(
// dataType: KafkatorioKeyedPacketData.KafkatorioKeyedPacketDataType,
//): String = packetTopic(KafkatorioPacketType.KEYED, dataType.name.lowercase())
//
//private fun packetTopic(
// packetType: KafkatorioPacketType,
// dataType: String,
//): String {
// return "$DOMAIN.packet.$packetType.$dataType"
//}
//sealed interface KafkatorioTopic {
//
// val type: String
// val entity: String
// val description: String
//
// companion object {
// private const val DOMAIN = "kafkatorio"
//
// val KafkatorioTopic.topic
// get() = listOf(DOMAIN, type, entity, description).joinToString(".")
//
// }
//
// data class Event(
// override val entity: String,
// override val description: String,
// ) : KafkatorioTopic {
// override val type: String = "event"
// }
//
//
// data class State(
// override val entity: String,
// override val description: String,
// ) : KafkatorioTopic {
// override val type: String = "state"
// }
//
//}

View file

@ -1,28 +1,11 @@
package dev.adamko.kafkatorio.processor.serdes
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import org.apache.kafka.common.serialization.Deserializer
import org.apache.kafka.common.serialization.Serde
import org.apache.kafka.common.serialization.Serializer
import org.http4k.format.ConfigurableKotlinxSerialization
import org.http4k.format.asConfigurable
import org.http4k.format.withStandardMappings
//object KafkatorioPacketSerde : Serde<KafkatorioPacket2> {
// override fun serializer() = Serializer<KafkatorioPacket> { _, message ->
// jsonMapper.encodeToString<KafkatorioPacket>(message).encodeToByteArray()
// }
//
// override fun deserializer() = Deserializer<KafkatorioPacket> { _, bytes ->
// jsonMapper.decodeFromString(bytes.decodeToString())
// }
//}
val jsonMapper: Json = Json {
prettyPrint = true
prettyPrintIndent = " "

View file

@ -4,7 +4,7 @@ package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.processor.admin.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.streams.map
@ -19,7 +19,7 @@ import org.apache.kafka.streams.processor.RecordContext
fun factorioServerPacketStream(
builder: StreamsBuilder = StreamsBuilder(),
): KStream<FactorioServerId, KafkatorioPacket2> {
): KStream<FactorioServerId, KafkatorioPacket> {
return builder.stream(
TOPIC_SRC_SERVER_LOG,
@ -29,14 +29,14 @@ fun factorioServerPacketStream(
Serdes.String(),
)
).map("decode-packets") { serverId: String, value: String ->
val packet = jsonMapper.decodeFromString<KafkatorioPacket2>(value)
val packet = jsonMapper.decodeFromString<KafkatorioPacket>(value)
FactorioServerId(serverId) to packet
}
}
fun splitFactorioServerPacketStream(
factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket2>
factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>
) {
factorioServerPacketStream
.to(
@ -45,7 +45,7 @@ fun splitFactorioServerPacketStream(
jsonMapper.serde(),
jsonMapper.serde(),
)
) { _: FactorioServerId, value: KafkatorioPacket2, _: RecordContext ->
) { _: FactorioServerId, value: KafkatorioPacket, _: RecordContext ->
// println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}")
value.data.topicName
}

View file

@ -10,8 +10,8 @@ import dev.adamko.kafkatorio.schema.common.MapTilePosition
import dev.adamko.kafkatorio.schema.common.MapTiles
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.toMapChunkPosition
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kotka.extensions.groupedAs
import dev.adamko.kotka.extensions.materializedWith
import dev.adamko.kotka.extensions.producedAs

View file

@ -2,8 +2,8 @@ package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.processor.admin.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema2.KafkatorioPacketData2
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.streams.mapValues
import dev.adamko.kotka.kxs.serde
@ -11,17 +11,17 @@ import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.KStream
inline fun <reified T : KafkatorioPacketData2> StreamsBuilder.streamPacketData(): KStream<FactorioServerId, T> {
inline fun <reified T : KafkatorioPacketData> StreamsBuilder.streamPacketData(): KStream<FactorioServerId, T> {
val simpleName = T::class.simpleName!!
val topicName = T::class.topicName
return stream<FactorioServerId, KafkatorioPacket2>(
return stream<FactorioServerId, KafkatorioPacket>(
topicName,
consumedAs("consume.${simpleName}", jsonMapper.serde(), jsonMapper.serde())
).filter { key: FactorioServerId?, value: KafkatorioPacket2? ->
).filter { key: FactorioServerId?, value: KafkatorioPacket? ->
key != null && value?.data is T
}.mapValues("consume.map-values.${simpleName}") { _: FactorioServerId, packet: KafkatorioPacket2 ->
}.mapValues("consume.map-values.${simpleName}") { _: FactorioServerId, packet: KafkatorioPacket ->
packet.data as T
}
}

View file

@ -3,8 +3,8 @@ package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.processor.WebsocketServer
import dev.adamko.kafkatorio.processor.admin.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema2.PlayerUpdate
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.kxs.serde
import kotlinx.serialization.encodeToString
@ -15,7 +15,7 @@ fun playerUpdatesToWsServer(
websocketServer: WebsocketServer,
builder: StreamsBuilder = StreamsBuilder(),
) {
builder.stream<FactorioServerId, KafkatorioPacket2>(
builder.stream<FactorioServerId, KafkatorioPacket>(
PlayerUpdate::class.topicName,
consumedAs("EventUpdate.Player-updates-for-ws-server", jsonMapper.serde(), jsonMapper.serde()),
).foreach { _, value ->

View file

@ -4,8 +4,8 @@ import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.MapTile
import dev.adamko.kafkatorio.schema.common.toHex
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kotka.extensions.materializedAs
import dev.adamko.kotka.extensions.streams.filter
import dev.adamko.kotka.extensions.streams.mapValues
@ -21,7 +21,7 @@ import org.apache.kafka.streams.kstream.KTable
@Serializable
value class TileProtoHashCode private constructor(val code: Int) {
constructor(tile: MapTile) : this(tile.proto.hashCode())
constructor(tile: FactorioPrototype2.MapTile) : this(tile.name.hashCode())
constructor(tile: FactorioPrototype.MapTile) : this(tile.name.hashCode())
}
@ -38,7 +38,7 @@ fun tileProtoColourDictionary(
return factorioServerPacketStream.mapValues("server-map-data.tile-prototypes.map-values") { _, protoPacket: PrototypesUpdate ->
val map = protoPacket
.prototypes
.filterIsInstance<FactorioPrototype2.MapTile>()
.filterIsInstance<FactorioPrototype.MapTile>()
.associate { TileProtoHashCode(it) to it.mapColour.toHex() }
TileColourDict(map)

View file

@ -12,11 +12,11 @@ import dev.adamko.kafkatorio.schema.common.MapTilePosition
import dev.adamko.kafkatorio.schema.common.PrototypeName
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.MapChunkUpdateKey
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate
import dev.adamko.kafkatorio.schema.packets.MapChunkUpdateKey
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kotka.kxs.serde
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec
@ -42,23 +42,23 @@ class GroupMapChunksTest : FunSpec({
val testDriver = TopologyTestDriver(topology)
val prototypesTestInputTopic: TestInputTopic<FactorioServerId, KafkatorioPacket2> =
val prototypesTestInputTopic: TestInputTopic<FactorioServerId, KafkatorioPacket> =
testDriver.createInputTopic(
prototypesTopicName,
jsonMapper.serde<FactorioServerId>().serializer(),
jsonMapper.serde<KafkatorioPacket2>().serializer(),
jsonMapper.serde<KafkatorioPacket>().serializer(),
)
val serverId = FactorioServerId("test-server-id")
prototypesTestInputTopic.pipeInput(
serverId,
KafkatorioPacket2(
KafkatorioPacket(
modVersion = "1.2.3",
tick = Tick(22u),
PrototypesUpdate(
listOf(
FactorioPrototype2.MapTile(
FactorioPrototype.MapTile(
PrototypeName("fake-proto"),
layer = 1u,
mapColour = Colour(0.3f, 0.2f, 0.1f, 1.0f),
@ -71,16 +71,16 @@ class GroupMapChunksTest : FunSpec({
)
)
val mapChunkUpdatesInputTopic: TestInputTopic<FactorioServerId, KafkatorioPacket2> =
val mapChunkUpdatesInputTopic: TestInputTopic<FactorioServerId, KafkatorioPacket> =
testDriver.createInputTopic(
mapChunkUpdatesTopicName,
jsonMapper.serde<FactorioServerId>().serializer(),
jsonMapper.serde<KafkatorioPacket2>().serializer(),
jsonMapper.serde<KafkatorioPacket>().serializer(),
)
mapChunkUpdatesInputTopic.pipeInput(
serverId,
KafkatorioPacket2(
KafkatorioPacket(
modVersion = "1.2.3",
tick = Tick(44u),
MapChunkUpdate(