- bump kafka sdk version
- fix kafka admin so it doesn't try making topics always - move some stuff to Kotka Streams - trying to optimise and debounce with windows... but it didn't work
This commit is contained in:
parent
f5855ba5bd
commit
f108431033
11 changed files with 216 additions and 135 deletions
|
|
@ -11,7 +11,7 @@ kotlin = "1.6.10"
|
||||||
## libs ##
|
## libs ##
|
||||||
camel-kafkaConnectors = "0.11.0"
|
camel-kafkaConnectors = "0.11.0"
|
||||||
http4k = "4.17.6.0"
|
http4k = "4.17.6.0"
|
||||||
kafka = "3.0.0"
|
kafka = "3.1.0"
|
||||||
kotlinx-serialization = "1.3.2"
|
kotlinx-serialization = "1.3.2"
|
||||||
kvision = "5.7.1"
|
kvision = "5.7.1"
|
||||||
log4j2 = "2.14.1"
|
log4j2 = "2.14.1"
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,8 @@ dependencies {
|
||||||
|
|
||||||
implementation("com.sksamuel.scrimage:scrimage-core:4.0.24")
|
implementation("com.sksamuel.scrimage:scrimage-core:4.0.24")
|
||||||
|
|
||||||
val kotkaVersion = "0.0.8"
|
// val kotkaVersion = "0.0.11"
|
||||||
|
val kotkaVersion = "store-iterators-SNAPSHOT"
|
||||||
implementation("com.github.adamko-dev.kotka-streams:kotka-streams:$kotkaVersion") {
|
implementation("com.github.adamko-dev.kotka-streams:kotka-streams:$kotkaVersion") {
|
||||||
isChanging = true
|
isChanging = true
|
||||||
}
|
}
|
||||||
|
|
@ -49,6 +50,10 @@ dependencies {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//configurations.all {
|
||||||
|
// resolutionStrategy.cacheDynamicVersionsFor(1, TimeUnit.MINUTES)
|
||||||
|
//}
|
||||||
|
|
||||||
application {
|
application {
|
||||||
mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
|
mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
|
||||||
}
|
}
|
||||||
|
|
@ -56,5 +61,6 @@ application {
|
||||||
tasks.withType<KotlinCompile> {
|
tasks.withType<KotlinCompile> {
|
||||||
kotlinOptions.freeCompilerArgs += listOf(
|
kotlinOptions.freeCompilerArgs += listOf(
|
||||||
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
|
"-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
|
||||||
|
"-opt-in=kotlinx.coroutines.FlowPreview",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,11 +28,10 @@ object CreateTopics {
|
||||||
|
|
||||||
fun create() {
|
fun create() {
|
||||||
|
|
||||||
KafkatorioPacket.PacketType.values().forEach { packetType ->
|
val currentTopics = currentTopics().map { it.name() }
|
||||||
|
|
||||||
val currentTopics = currentTopics().map { it.name() }
|
val kafkatorioTopics = buildSet {
|
||||||
|
KafkatorioPacket.PacketType.values().forEach { packetType ->
|
||||||
val kafkatorioTopics = buildSet {
|
|
||||||
add(KafkatorioTopology.sourceTopic)
|
add(KafkatorioTopology.sourceTopic)
|
||||||
when (packetType) {
|
when (packetType) {
|
||||||
KafkatorioPacket.PacketType.EVENT ->
|
KafkatorioPacket.PacketType.EVENT ->
|
||||||
|
|
@ -44,30 +43,30 @@ object CreateTopics {
|
||||||
KafkatorioPacket.PacketType.PROTOTYPES ->
|
KafkatorioPacket.PacketType.PROTOTYPES ->
|
||||||
add("kafkatorio.${packetType.name}.all")
|
add("kafkatorio.${packetType.name}.all")
|
||||||
}
|
}
|
||||||
}.minus(currentTopics.toSet())
|
|
||||||
|
|
||||||
if (currentTopics.isNotEmpty()) {
|
|
||||||
|
|
||||||
logger.info("Creating ${kafkatorioTopics.size} topics")
|
|
||||||
val result = kafkaAdmin.createTopics { kafkatorioTopics }
|
|
||||||
|
|
||||||
// wait for all brokers to become aware of the created topics
|
|
||||||
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
|
||||||
|
|
||||||
result.values().forEach {
|
|
||||||
logger.info("Created topic: ${it.key} ${it.value.get()}")
|
|
||||||
}
|
|
||||||
|
|
||||||
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
|
||||||
|
|
||||||
logger.info("Listing topics")
|
|
||||||
kafkaAdmin.listTopics()
|
|
||||||
.listings()
|
|
||||||
.get()
|
|
||||||
.forEach {
|
|
||||||
logger.info("Topic: $it")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}.minus(currentTopics.toSet())
|
||||||
|
|
||||||
|
if (kafkatorioTopics.isNotEmpty()) {
|
||||||
|
|
||||||
|
logger.info("Creating ${kafkatorioTopics.size} topics")
|
||||||
|
val result = kafkaAdmin.createTopics { kafkatorioTopics }
|
||||||
|
|
||||||
|
// wait for all brokers to become aware of the created topics
|
||||||
|
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
||||||
|
|
||||||
|
result.values().forEach {
|
||||||
|
logger.info("Created topic: ${it.key} ${it.value.get()}")
|
||||||
|
}
|
||||||
|
|
||||||
|
Thread.sleep(TimeUnit.SECONDS.toMillis(5))
|
||||||
|
|
||||||
|
logger.info("Listing topics")
|
||||||
|
kafkaAdmin.listTopics()
|
||||||
|
.listings()
|
||||||
|
.get()
|
||||||
|
.forEach {
|
||||||
|
logger.info("Topic: $it")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
package dev.adamko.kafkatorio.processor
|
package dev.adamko.kafkatorio.processor
|
||||||
|
|
||||||
|
import dev.adamko.kafkatorio.events.schema.Colour
|
||||||
import dev.adamko.kafkatorio.events.schema.MapTile
|
import dev.adamko.kafkatorio.events.schema.MapTile
|
||||||
import dev.adamko.kafkatorio.events.schema.MapTilePrototype
|
|
||||||
import dev.adamko.kafkatorio.processor.topology.PrototypeName
|
import dev.adamko.kafkatorio.processor.topology.PrototypeName
|
||||||
import dev.adamko.kafkatorio.processor.topology.TileUpdateRecordKey
|
import dev.adamko.kafkatorio.processor.topology.TileUpdateRecordKey
|
||||||
import dev.adamko.kafkatorio.processor.topology.WebMapTileChunkPixels
|
import dev.adamko.kafkatorio.processor.topology.WebMapTileChunkPixels
|
||||||
|
|
@ -9,9 +9,9 @@ import dev.adamko.kafkatorio.processor.topology.WebMapTileChunkPosition
|
||||||
import dev.adamko.kafkatorio.processor.topology.aggregateWebMapTiles
|
import dev.adamko.kafkatorio.processor.topology.aggregateWebMapTiles
|
||||||
import dev.adamko.kafkatorio.processor.topology.allMapTilesTable
|
import dev.adamko.kafkatorio.processor.topology.allMapTilesTable
|
||||||
import dev.adamko.kafkatorio.processor.topology.playerUpdatesToWsServer
|
import dev.adamko.kafkatorio.processor.topology.playerUpdatesToWsServer
|
||||||
import dev.adamko.kafkatorio.processor.topology.prototypesTable
|
|
||||||
import dev.adamko.kafkatorio.processor.topology.saveTileImages
|
import dev.adamko.kafkatorio.processor.topology.saveTileImages
|
||||||
import dev.adamko.kafkatorio.processor.topology.splitFactorioServerLog
|
import dev.adamko.kafkatorio.processor.topology.splitFactorioServerLog
|
||||||
|
import dev.adamko.kafkatorio.processor.topology.tilePrototypeColourTable
|
||||||
import java.time.Duration
|
import java.time.Duration
|
||||||
import org.apache.kafka.streams.KafkaStreams
|
import org.apache.kafka.streams.KafkaStreams
|
||||||
import org.apache.kafka.streams.StreamsBuilder
|
import org.apache.kafka.streams.StreamsBuilder
|
||||||
|
|
@ -35,10 +35,10 @@ class KafkatorioTopology(
|
||||||
playerUpdatesToWsServer(websocketServer, builder)
|
playerUpdatesToWsServer(websocketServer, builder)
|
||||||
|
|
||||||
val allMapTilesTable: KTable<TileUpdateRecordKey, MapTile> = allMapTilesTable(builder)
|
val allMapTilesTable: KTable<TileUpdateRecordKey, MapTile> = allMapTilesTable(builder)
|
||||||
val tilePrototypesTable: KTable<PrototypeName, MapTilePrototype> = prototypesTable(builder)
|
val tilePrototypeColourTable: KTable<PrototypeName, Colour> = tilePrototypeColourTable(builder)
|
||||||
|
|
||||||
val webMapTiles: KTable<WebMapTileChunkPosition, WebMapTileChunkPixels> =
|
val webMapTiles: KTable<WebMapTileChunkPosition, WebMapTileChunkPixels> =
|
||||||
aggregateWebMapTiles(allMapTilesTable, tilePrototypesTable)
|
aggregateWebMapTiles(allMapTilesTable, tilePrototypeColourTable)
|
||||||
|
|
||||||
saveTileImages(webMapTiles)
|
saveTileImages(webMapTiles)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,44 +7,44 @@ import org.apache.kafka.common.serialization.Deserializer
|
||||||
import org.apache.kafka.common.serialization.Serde
|
import org.apache.kafka.common.serialization.Serde
|
||||||
import org.apache.kafka.common.serialization.Serializer
|
import org.apache.kafka.common.serialization.Serializer
|
||||||
|
|
||||||
|
//
|
||||||
inline fun <reified T> BinaryFormat.kafkaSerializer() =
|
//inline fun <reified T> BinaryFormat.kafkaSerializer() =
|
||||||
Serializer { topic: String, data: T ->
|
// Serializer { topic: String, data: T ->
|
||||||
runCatching {
|
// runCatching {
|
||||||
encodeToByteArray(data)
|
// encodeToByteArray(data)
|
||||||
}.getOrElse { e ->
|
// }.getOrElse { e ->
|
||||||
println(
|
// println(
|
||||||
"""
|
// """
|
||||||
Exception on encodeToString
|
// Exception on encodeToString
|
||||||
Topic: $topic
|
// Topic: $topic
|
||||||
topicData: $data
|
// topicData: $data
|
||||||
topicData as T: $data
|
// topicData as T: $data
|
||||||
""".trimIndent()
|
// """.trimIndent()
|
||||||
)
|
// )
|
||||||
e.printStackTrace()
|
// e.printStackTrace()
|
||||||
throw e
|
// throw e
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
inline fun <reified T> BinaryFormat.kafkaDeserializer() =
|
//inline fun <reified T> BinaryFormat.kafkaDeserializer() =
|
||||||
Deserializer { topic: String, data: ByteArray ->
|
// Deserializer { topic: String, data: ByteArray ->
|
||||||
runCatching {
|
// runCatching {
|
||||||
decodeFromByteArray<T>(data)
|
// decodeFromByteArray<T>(data)
|
||||||
}.getOrElse { e ->
|
// }.getOrElse { e ->
|
||||||
println(
|
// println(
|
||||||
"""
|
// """
|
||||||
Exception on decodeFromString,
|
// Exception on decodeFromString,
|
||||||
Topic: $topic
|
// Topic: $topic
|
||||||
topicData: $data
|
// topicData: $data
|
||||||
topicData as T: ${data as? T}
|
// topicData as T: ${data as? T}
|
||||||
""".trimIndent()
|
// """.trimIndent()
|
||||||
)
|
// )
|
||||||
e.printStackTrace()
|
// e.printStackTrace()
|
||||||
throw e
|
// throw e
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
inline fun <reified T> BinaryFormat.serde() = object : Serde<T> {
|
//inline fun <reified T> BinaryFormat.serde() = object : Serde<T> {
|
||||||
override fun serializer(): Serializer<T> = kafkaSerializer()
|
// override fun serializer(): Serializer<T> = kafkaSerializer()
|
||||||
override fun deserializer(): Deserializer<T> = kafkaDeserializer()
|
// override fun deserializer(): Deserializer<T> = kafkaDeserializer()
|
||||||
}
|
//}
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,6 @@ import dev.adamko.kafkatorio.events.schema.MapTilePosition
|
||||||
import dev.adamko.kafkatorio.events.schema.MapTiles
|
import dev.adamko.kafkatorio.events.schema.MapTiles
|
||||||
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
|
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
|
||||||
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
|
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
|
||||||
import dev.adamko.kafkatorio.processor.serdes.serde
|
|
||||||
import dev.adamko.kotka.extensions.consumedAs
|
import dev.adamko.kotka.extensions.consumedAs
|
||||||
import dev.adamko.kotka.extensions.materializedWith
|
import dev.adamko.kotka.extensions.materializedWith
|
||||||
import dev.adamko.kotka.extensions.streams.filter
|
import dev.adamko.kotka.extensions.streams.filter
|
||||||
|
|
@ -75,12 +74,5 @@ fun allMapTilesTable(
|
||||||
"all-tiles-keyed-on-position",
|
"all-tiles-keyed-on-position",
|
||||||
materializedWith(kxsBinary.serde(), kxsBinary.serde())
|
materializedWith(kxsBinary.serde(), kxsBinary.serde())
|
||||||
)
|
)
|
||||||
.suppress(
|
|
||||||
Suppressed.untilTimeLimit(
|
|
||||||
Duration.ofSeconds(30),
|
|
||||||
Suppressed.BufferConfig
|
|
||||||
.maxRecords(30)
|
|
||||||
.emitEarlyWhenFull()
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
package dev.adamko.kafkatorio.processor.topology
|
package dev.adamko.kafkatorio.processor.topology
|
||||||
|
|
||||||
|
import com.sksamuel.scrimage.color.RGBColor
|
||||||
|
import dev.adamko.kafkatorio.events.schema.Colour
|
||||||
|
import kotlin.math.roundToInt
|
||||||
import kotlinx.serialization.Serializable
|
import kotlinx.serialization.Serializable
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
|
|
@ -15,3 +18,12 @@ value class SurfaceIndex(
|
||||||
value class FactorioServerId(val id: String) {
|
value class FactorioServerId(val id: String) {
|
||||||
override fun toString() = id
|
override fun toString() = id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fun Colour.toRgbColor(): RGBColor {
|
||||||
|
return RGBColor(
|
||||||
|
red.roundToInt(),
|
||||||
|
green.roundToInt(),
|
||||||
|
blue.roundToInt(),
|
||||||
|
alpha.roundToInt(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ fun playerUpdatesToWsServer(
|
||||||
consumedAs("stream-player-updates-for-ws-server", jsonMapper.serde(), jsonMapper.serde())
|
consumedAs("stream-player-updates-for-ws-server", jsonMapper.serde(), jsonMapper.serde())
|
||||||
)
|
)
|
||||||
.foreach { _, value ->
|
.foreach { _, value ->
|
||||||
println("sending ${value.packetType} packet to websocket")
|
// println("sending ${value.packetType} packet to websocket")
|
||||||
websocketServer.sendMessage(jsonMapper.encodeToString(value))
|
websocketServer.sendMessage(jsonMapper.encodeToString(value))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,12 @@
|
||||||
package dev.adamko.kafkatorio.processor.topology
|
package dev.adamko.kafkatorio.processor.topology
|
||||||
|
|
||||||
|
import com.sksamuel.scrimage.color.RGBColor
|
||||||
|
import dev.adamko.kafkatorio.events.schema.Colour
|
||||||
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
|
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
|
||||||
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
|
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
|
||||||
import dev.adamko.kafkatorio.events.schema.MapTilePrototype
|
import dev.adamko.kafkatorio.events.schema.MapTilePrototype
|
||||||
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
|
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
|
||||||
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
|
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
|
||||||
import dev.adamko.kafkatorio.processor.serdes.serde
|
|
||||||
import dev.adamko.kotka.extensions.consumedAs
|
import dev.adamko.kotka.extensions.consumedAs
|
||||||
import dev.adamko.kotka.extensions.materializedWith
|
import dev.adamko.kotka.extensions.materializedWith
|
||||||
import dev.adamko.kotka.extensions.streams.flatMap
|
import dev.adamko.kotka.extensions.streams.flatMap
|
||||||
|
|
@ -23,7 +24,31 @@ value class PrototypeName(val name: String) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
fun prototypesTable(builder: StreamsBuilder): KTable<PrototypeName, MapTilePrototype> {
|
//fun prototypesTable(builder: StreamsBuilder): KTable<PrototypeName, MapTilePrototype> {
|
||||||
|
// return builder.stream<FactorioPacketKey, FactorioPrototypes>(
|
||||||
|
// "kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
|
||||||
|
// consumedAs(
|
||||||
|
// "consume-all-prototypes-packets",
|
||||||
|
// jsonMapper.serde(),
|
||||||
|
// jsonMapper.serde()
|
||||||
|
// )
|
||||||
|
// )
|
||||||
|
// .flatMap("map-MapTilePrototype") { _, prototypes: FactorioPrototypes ->
|
||||||
|
// prototypes.prototypes
|
||||||
|
// .filterIsInstance<MapTilePrototype>()
|
||||||
|
// .map { PrototypeName(it.name) to it }
|
||||||
|
// }
|
||||||
|
// .toTable(
|
||||||
|
// materializedWith(
|
||||||
|
// kxsBinary.serde(),
|
||||||
|
// kxsBinary.serde()
|
||||||
|
// )
|
||||||
|
// )
|
||||||
|
//}
|
||||||
|
|
||||||
|
|
||||||
|
/** Get the [Colour] of each [MapTilePrototype] */
|
||||||
|
fun tilePrototypeColourTable(builder: StreamsBuilder): KTable<PrototypeName, Colour> {
|
||||||
return builder.stream<FactorioPacketKey, FactorioPrototypes>(
|
return builder.stream<FactorioPacketKey, FactorioPrototypes>(
|
||||||
"kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
|
"kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
|
||||||
consumedAs(
|
consumedAs(
|
||||||
|
|
@ -35,7 +60,9 @@ fun prototypesTable(builder: StreamsBuilder): KTable<PrototypeName, MapTileProto
|
||||||
.flatMap("map-MapTilePrototype") { _, prototypes: FactorioPrototypes ->
|
.flatMap("map-MapTilePrototype") { _, prototypes: FactorioPrototypes ->
|
||||||
prototypes.prototypes
|
prototypes.prototypes
|
||||||
.filterIsInstance<MapTilePrototype>()
|
.filterIsInstance<MapTilePrototype>()
|
||||||
.map { PrototypeName(it.name) to it }
|
.map {
|
||||||
|
PrototypeName(it.name) to it.mapColour
|
||||||
|
}
|
||||||
}
|
}
|
||||||
.toTable(
|
.toTable(
|
||||||
materializedWith(
|
materializedWith(
|
||||||
|
|
|
||||||
|
|
@ -1,28 +1,37 @@
|
||||||
package dev.adamko.kafkatorio.processor.topology
|
package dev.adamko.kafkatorio.processor.topology
|
||||||
|
|
||||||
|
import com.sksamuel.scrimage.color.RGBColor
|
||||||
import dev.adamko.kafkatorio.events.schema.Colour
|
import dev.adamko.kafkatorio.events.schema.Colour
|
||||||
import dev.adamko.kafkatorio.events.schema.MapTile
|
import dev.adamko.kafkatorio.events.schema.MapTile
|
||||||
import dev.adamko.kafkatorio.events.schema.MapTilePosition
|
import dev.adamko.kafkatorio.events.schema.MapTilePosition
|
||||||
import dev.adamko.kafkatorio.events.schema.MapTilePrototype
|
|
||||||
import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
|
import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
|
||||||
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
|
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
|
||||||
import dev.adamko.kafkatorio.processor.serdes.serde
|
|
||||||
import dev.adamko.kotka.extensions.groupedAs
|
import dev.adamko.kotka.extensions.groupedAs
|
||||||
import dev.adamko.kotka.extensions.materializedAs
|
import dev.adamko.kotka.extensions.materializedAs
|
||||||
|
import dev.adamko.kotka.extensions.materializedWith
|
||||||
import dev.adamko.kotka.extensions.namedAs
|
import dev.adamko.kotka.extensions.namedAs
|
||||||
|
import dev.adamko.kotka.extensions.streams.groupBy
|
||||||
import dev.adamko.kotka.extensions.tables.groupBy
|
import dev.adamko.kotka.extensions.tables.groupBy
|
||||||
import dev.adamko.kotka.extensions.tables.join
|
import dev.adamko.kotka.extensions.tables.join
|
||||||
import dev.adamko.kotka.extensions.tables.mapValues
|
import dev.adamko.kotka.extensions.tables.mapValues
|
||||||
|
import dev.adamko.kotka.extensions.tables.toStream
|
||||||
import dev.adamko.kotka.extensions.toKeyValue
|
import dev.adamko.kotka.extensions.toKeyValue
|
||||||
|
import dev.adamko.kotka.kxs.serde
|
||||||
|
import java.awt.Color
|
||||||
import java.time.Duration
|
import java.time.Duration
|
||||||
import kotlinx.serialization.Serializable
|
import kotlinx.serialization.Serializable
|
||||||
import org.apache.kafka.streams.kstream.KTable
|
import org.apache.kafka.streams.kstream.KTable
|
||||||
|
import org.apache.kafka.streams.kstream.SessionWindows
|
||||||
import org.apache.kafka.streams.kstream.Suppressed
|
import org.apache.kafka.streams.kstream.Suppressed
|
||||||
|
import org.apache.kafka.streams.kstream.Windowed
|
||||||
|
|
||||||
|
|
||||||
/** The tiles of the webmap will be this many pixels high/wide */
|
/** The tiles of the webmap will be this many pixels high/wide */
|
||||||
const val WEB_MAP_IMAGE_TILE_SIZE = 256
|
const val WEB_MAP_IMAGE_TILE_SIZE = 256
|
||||||
|
|
||||||
|
/** Debounce duration */
|
||||||
|
private val WINDOW_INACTIVITY_DURATION = Duration.ofSeconds(60)
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
data class WebMapTileChunkPosition(
|
data class WebMapTileChunkPosition(
|
||||||
val chunkSize: Int,
|
val chunkSize: Int,
|
||||||
|
|
@ -32,31 +41,52 @@ data class WebMapTileChunkPosition(
|
||||||
)
|
)
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
data class WebMapTilePixel(
|
private data class WebMapTilePixel(
|
||||||
val mapColour: Colour,
|
val mapColour: Colour,
|
||||||
val tilePosition: MapTilePosition,
|
val tilePosition: MapTilePosition,
|
||||||
)
|
)
|
||||||
|
|
||||||
@Serializable
|
@Serializable
|
||||||
data class WebMapTileChunkPixels(
|
@JvmInline
|
||||||
val pixels: Set<WebMapTilePixel> = mutableSetOf(),
|
value class WebMapTileChunkPixels(
|
||||||
|
val pixels: Map<MapTilePosition, Colour>,
|
||||||
)
|
)
|
||||||
|
|
||||||
/** Temporary mutable aggregator */
|
/** Temporary mutable accumulator */
|
||||||
@Serializable
|
@Serializable
|
||||||
private data class WebMapTileChunkPixelsAcc(
|
@JvmInline
|
||||||
val pixels: MutableSet<WebMapTilePixel> = mutableSetOf(),
|
private value class WebMapTileChunkPixelsAcc(
|
||||||
)
|
val pixels: MutableMap<MapTilePosition, Colour> = mutableMapOf(),
|
||||||
|
) {
|
||||||
|
|
||||||
|
operator fun plus(pixel: WebMapTilePixel): WebMapTileChunkPixelsAcc {
|
||||||
|
pixels[pixel.tilePosition] = pixel.mapColour
|
||||||
|
return this
|
||||||
|
}
|
||||||
|
|
||||||
|
operator fun plus(other: WebMapTileChunkPixelsAcc): WebMapTileChunkPixelsAcc {
|
||||||
|
this.pixels += other.pixels
|
||||||
|
return this
|
||||||
|
}
|
||||||
|
|
||||||
|
operator fun minus(other: WebMapTileChunkPixelsAcc): WebMapTileChunkPixelsAcc {
|
||||||
|
this.pixels -= other.pixels.keys
|
||||||
|
return this
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
fun aggregateWebMapTiles(
|
fun aggregateWebMapTiles(
|
||||||
allMapTilesTable: KTable<TileUpdateRecordKey, MapTile>,
|
allMapTilesTable: KTable<TileUpdateRecordKey, MapTile>,
|
||||||
tilePrototypesTable: KTable<PrototypeName, MapTilePrototype>,
|
tilePrototypeColourTable: KTable<PrototypeName, Colour>,
|
||||||
): KTable<WebMapTileChunkPosition, WebMapTileChunkPixels> {
|
): KTable<WebMapTileChunkPosition, WebMapTileChunkPixels> {
|
||||||
|
|
||||||
val webmapTileColours: KTable<TileUpdateRecordKey, WebMapTilePixel> =
|
|
||||||
|
val pixels: KTable<TileUpdateRecordKey, WebMapTilePixel> =
|
||||||
allMapTilesTable
|
allMapTilesTable
|
||||||
.join(
|
.join(
|
||||||
tilePrototypesTable,
|
tilePrototypeColourTable,
|
||||||
"convert-tiles-to-colours",
|
"convert-tiles-to-colours",
|
||||||
materializedAs(
|
materializedAs(
|
||||||
"store-web-map-tile-pixels",
|
"store-web-map-tile-pixels",
|
||||||
|
|
@ -64,13 +94,14 @@ fun aggregateWebMapTiles(
|
||||||
kxsBinary.serde(),
|
kxsBinary.serde(),
|
||||||
),
|
),
|
||||||
{ PrototypeName(it.prototypeName) },
|
{ PrototypeName(it.prototypeName) },
|
||||||
) { tile: MapTile, proto: MapTilePrototype ->
|
) { tile: MapTile, colour: Colour ->
|
||||||
WebMapTilePixel(proto.mapColour, tile.position)
|
WebMapTilePixel(colour, tile.position)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
val webMapTileChunkAggregate: KTable<WebMapTileChunkPosition, WebMapTileChunkPixelsAcc> =
|
val chunkedPixelsWindowed: KTable<Windowed<WebMapTileChunkPosition>, WebMapTileChunkPixelsAcc> =
|
||||||
webmapTileColours
|
pixels
|
||||||
|
.toStream("stream-webmap-tile-pixels")
|
||||||
.groupBy(
|
.groupBy(
|
||||||
groupedAs(
|
groupedAs(
|
||||||
"group-webmap-tile-colours-into-chunks",
|
"group-webmap-tile-colours-into-chunks",
|
||||||
|
|
@ -85,17 +116,21 @@ fun aggregateWebMapTiles(
|
||||||
chunkY,
|
chunkY,
|
||||||
pos.surfaceIndex
|
pos.surfaceIndex
|
||||||
)
|
)
|
||||||
(chunkPos to px).toKeyValue()
|
chunkPos
|
||||||
}
|
}
|
||||||
|
.windowedBy(
|
||||||
|
SessionWindows.ofInactivityGapAndGrace(
|
||||||
|
WINDOW_INACTIVITY_DURATION,
|
||||||
|
WINDOW_INACTIVITY_DURATION,
|
||||||
|
)
|
||||||
|
)
|
||||||
.aggregate(
|
.aggregate(
|
||||||
{ WebMapTileChunkPixelsAcc() },
|
{ WebMapTileChunkPixelsAcc() },
|
||||||
/* adder */
|
|
||||||
{ _: WebMapTileChunkPosition, newValue: WebMapTilePixel, acc: WebMapTileChunkPixelsAcc ->
|
{ _: WebMapTileChunkPosition, newValue: WebMapTilePixel, acc: WebMapTileChunkPixelsAcc ->
|
||||||
acc.apply { pixels.add(newValue) }
|
acc + newValue
|
||||||
},
|
},
|
||||||
/* subtractor */
|
{ _, aggOne: WebMapTileChunkPixelsAcc, aggTwo: WebMapTileChunkPixelsAcc ->
|
||||||
{ _: WebMapTileChunkPosition, oldValue: WebMapTilePixel, acc: WebMapTileChunkPixelsAcc ->
|
aggOne + aggTwo
|
||||||
acc.apply { pixels.minus(oldValue) }
|
|
||||||
},
|
},
|
||||||
namedAs("web-map-tile-colours-into-$WEB_MAP_IMAGE_TILE_SIZE-chunks"),
|
namedAs("web-map-tile-colours-into-$WEB_MAP_IMAGE_TILE_SIZE-chunks"),
|
||||||
materializedAs(
|
materializedAs(
|
||||||
|
|
@ -105,20 +140,34 @@ fun aggregateWebMapTiles(
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
.suppress(
|
.suppress(
|
||||||
Suppressed.untilTimeLimit<WebMapTileChunkPosition>(
|
Suppressed.untilWindowCloses(
|
||||||
Duration.ofSeconds(30),
|
Suppressed.BufferConfig.unbounded()
|
||||||
Suppressed.BufferConfig
|
).withName("chunked-pixels-debounce")
|
||||||
.maxRecords(30)
|
|
||||||
// .withMaxBytes(31457280 ) // 30MB
|
|
||||||
.emitEarlyWhenFull()
|
|
||||||
).withName("web-map-tile-aggregate-debounce")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return webMapTileChunkAggregate
|
|
||||||
|
// merge windows
|
||||||
|
val chunkedPixels: KTable<WebMapTileChunkPosition, WebMapTileChunkPixelsAcc> =
|
||||||
|
chunkedPixelsWindowed
|
||||||
|
.groupBy(
|
||||||
|
groupedAs("merge-chunked-pixels.group-keys", kxsBinary.serde(), kxsBinary.serde())
|
||||||
|
) { a: Windowed<WebMapTileChunkPosition>, b: WebMapTileChunkPixelsAcc ->
|
||||||
|
(a.key() to b).toKeyValue()
|
||||||
|
}
|
||||||
|
.reduce(
|
||||||
|
/* adder */
|
||||||
|
{ pixels1, pixels2 -> pixels1 + pixels2 },
|
||||||
|
/* subtractor */
|
||||||
|
{ pixels1, pixels2 -> pixels1 - pixels2 },
|
||||||
|
namedAs("merge-chunked-pixels.reduce-values"),
|
||||||
|
materializedAs("merge-chunked-pixels.reduce-values.store", kxsBinary.serde(), kxsBinary.serde())
|
||||||
|
)
|
||||||
|
|
||||||
|
return chunkedPixels
|
||||||
.mapValues(
|
.mapValues(
|
||||||
"finalise-web-map-tile-colour-chunk-aggregation",
|
"finalise-web-map-tile-colour-chunk-aggregation",
|
||||||
materializedAs("web-map-tile-colour-chunks", kxsBinary.serde(), kxsBinary.serde())
|
materializedAs("web-map-tile-colour-chunks", kxsBinary.serde(), kxsBinary.serde())
|
||||||
) { _, v ->
|
) { _: WebMapTileChunkPosition, v: WebMapTileChunkPixelsAcc ->
|
||||||
WebMapTileChunkPixels(v.pixels)
|
WebMapTileChunkPixels(v.pixels)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,28 +3,24 @@ package dev.adamko.kafkatorio.processor.topology
|
||||||
import com.sksamuel.scrimage.ImmutableImage
|
import com.sksamuel.scrimage.ImmutableImage
|
||||||
import com.sksamuel.scrimage.color.RGBColor
|
import com.sksamuel.scrimage.color.RGBColor
|
||||||
import com.sksamuel.scrimage.nio.PngWriter
|
import com.sksamuel.scrimage.nio.PngWriter
|
||||||
|
import dev.adamko.kafkatorio.events.schema.Colour
|
||||||
|
import dev.adamko.kafkatorio.events.schema.MapTilePosition
|
||||||
import dev.adamko.kotka.extensions.tables.toStream
|
import dev.adamko.kotka.extensions.tables.toStream
|
||||||
import java.awt.Color
|
import java.awt.Color
|
||||||
import java.awt.image.BufferedImage
|
import java.awt.image.BufferedImage
|
||||||
import java.io.File
|
import java.io.File
|
||||||
import java.time.Duration
|
|
||||||
import kotlin.math.abs
|
import kotlin.math.abs
|
||||||
import kotlin.math.roundToInt
|
import kotlin.math.roundToInt
|
||||||
import org.apache.kafka.streams.kstream.KTable
|
import org.apache.kafka.streams.kstream.KTable
|
||||||
import org.apache.kafka.streams.kstream.Suppressed
|
|
||||||
|
|
||||||
|
|
||||||
|
private val COLOUR_TRANSPARENT = RGBColor.fromAwt(Color(0f, 0f, 0f, 0f))
|
||||||
|
|
||||||
fun saveTileImages(
|
fun saveTileImages(
|
||||||
webMapTiles: KTable<WebMapTileChunkPosition, WebMapTileChunkPixels>
|
webMapTiles: KTable<WebMapTileChunkPosition, WebMapTileChunkPixels>
|
||||||
) {
|
) {
|
||||||
|
|
||||||
webMapTiles
|
webMapTiles
|
||||||
.suppress(
|
|
||||||
Suppressed.untilTimeLimit<WebMapTileChunkPosition>(
|
|
||||||
Duration.ofSeconds(30),
|
|
||||||
Suppressed.BufferConfig.maxRecords(30)
|
|
||||||
).withName("save-tile-images-debounce")
|
|
||||||
)
|
|
||||||
.toStream("get-webmap-tile-updates")
|
.toStream("get-webmap-tile-updates")
|
||||||
.foreach { position: WebMapTileChunkPosition, pixels: WebMapTileChunkPixels ->
|
.foreach { position: WebMapTileChunkPosition, pixels: WebMapTileChunkPixels ->
|
||||||
runCatching {
|
runCatching {
|
||||||
|
|
@ -42,7 +38,7 @@ fun saveTileImages(
|
||||||
|
|
||||||
private fun saveMapTilesPng(
|
private fun saveMapTilesPng(
|
||||||
chunkPosition: WebMapTileChunkPosition,
|
chunkPosition: WebMapTileChunkPosition,
|
||||||
pixels: Set<WebMapTilePixel>
|
pixels: Map<MapTilePosition, Colour>,
|
||||||
) {
|
) {
|
||||||
val chunkOriginX: Int = chunkPosition.x * chunkPosition.chunkSize // - 1
|
val chunkOriginX: Int = chunkPosition.x * chunkPosition.chunkSize // - 1
|
||||||
val chunkOriginY: Int = chunkPosition.y * chunkPosition.chunkSize // - 1
|
val chunkOriginY: Int = chunkPosition.y * chunkPosition.chunkSize // - 1
|
||||||
|
|
@ -52,21 +48,21 @@ private fun saveMapTilesPng(
|
||||||
ImmutableImage.filled(
|
ImmutableImage.filled(
|
||||||
chunkPosition.chunkSize,
|
chunkPosition.chunkSize,
|
||||||
chunkPosition.chunkSize,
|
chunkPosition.chunkSize,
|
||||||
Color.BLACK,
|
COLOUR_TRANSPARENT.awt(),
|
||||||
BufferedImage.TYPE_INT_ARGB
|
BufferedImage.TYPE_INT_ARGB
|
||||||
)
|
)
|
||||||
|
|
||||||
pixels.forEach { pixel ->
|
pixels.forEach { (tilePosition, colour) ->
|
||||||
|
|
||||||
val rgbColour = RGBColor(
|
val rgbColour = RGBColor(
|
||||||
pixel.mapColour.red.roundToInt(),
|
colour.red.roundToInt(),
|
||||||
pixel.mapColour.green.roundToInt(),
|
colour.green.roundToInt(),
|
||||||
pixel.mapColour.blue.roundToInt(),
|
colour.blue.roundToInt(),
|
||||||
pixel.mapColour.alpha.roundToInt(),
|
colour.alpha.roundToInt(),
|
||||||
)
|
)
|
||||||
|
|
||||||
val pixelX = abs(abs(pixel.tilePosition.x) - abs(chunkOriginX))
|
val pixelX = abs(abs(tilePosition.x) - abs(chunkOriginX))
|
||||||
val pixelY = abs(abs(pixel.tilePosition.y) - abs(chunkOriginY))
|
val pixelY = abs(abs(tilePosition.y) - abs(chunkOriginY))
|
||||||
|
|
||||||
chunkImage.setColor(
|
chunkImage.setColor(
|
||||||
pixelX,
|
pixelX,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue