diff --git a/README.md b/README.md index 022a03a..43670cc 100644 --- a/README.md +++ b/README.md @@ -58,3 +58,7 @@ https://www.npmjs.com/package/wasm2lua?activeTab=readme * CBOR * https://www.zash.se/lua-cbor.html * https://code.zash.se/lua-cbor/archive/68b3a36d0816.tar.gz + +* BSON + * https://github.com/mpaland/bsonfy + * https://github.com/tcoram/bson-lua diff --git a/build.gradle.kts b/build.gradle.kts index fd005aa..fcb9c04 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,7 +6,7 @@ plugins { } group = "dev.adamko.kafkatorio" -version = "0.2.2" +version = "0.2.3" val licenseFile: RegularFile by extra(layout.projectDirectory.file("LICENSE")) diff --git a/modules/events-mod/src/main/typescript/generated/kafkatorio-schema/prototype.d.ts b/modules/events-mod/src/main/typescript/generated/kafkatorio-schema/prototype.d.ts index e4d4067..67464d8 100644 --- a/modules/events-mod/src/main/typescript/generated/kafkatorio-schema/prototype.d.ts +++ b/modules/events-mod/src/main/typescript/generated/kafkatorio-schema/prototype.d.ts @@ -6,11 +6,11 @@ interface FactorioPrototype { prototypeObjectName: PrototypeObjectName; } -interface FactorioMapTilePrototype extends FactorioPrototype { +interface MapTilePrototype extends FactorioPrototype { canBeMined: boolean; collisionMasks: string[]; layer: uint; - mapColor: Colour; + mapColour: Colour; name: string; order: string; prototypeObjectName: PrototypeObjectName; diff --git a/modules/events-mod/src/main/typescript/main/config-update/convertPrototypes.ts b/modules/events-mod/src/main/typescript/main/config-update/convertPrototypes.ts index a3b803f..b9c6203 100644 --- a/modules/events-mod/src/main/typescript/main/config-update/convertPrototypes.ts +++ b/modules/events-mod/src/main/typescript/main/config-update/convertPrototypes.ts @@ -8,18 +8,18 @@ export function convertPrototypes(): FactorioPrototype[] { return prototypes } -function getMapTilePrototypes(): FactorioMapTilePrototype[] { - let tiles: FactorioMapTilePrototype[] = [] +function getMapTilePrototypes(): MapTilePrototype[] { + let tiles: MapTilePrototype[] = [] for (let [, tile] of pairs(game.tile_prototypes)) { tiles.push( - { + { prototypeObjectName: tile.object_name, name: tile.name, order: tile.order, layer: tile.layer, collisionMasks: convertCollisionMaskToNames(tile.collision_mask), - mapColor: Converters.mapColour(tile.map_color), + mapColour: Converters.mapColour(tile.map_color), canBeMined: tile.mineable_properties.minable, } ) diff --git a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt index 8633fc3..1c4854b 100644 --- a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt +++ b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt @@ -54,13 +54,19 @@ data class MapTilePosition( val y: Int, ) { constructor(x: Number, y: Number) : this(x.toInt(), y.toInt()) + + fun toMapChunkPosition() = MapChunkPosition( + x / MAP_CHUNK_SIZE, + y / MAP_CHUNK_SIZE, + ) + } /** * Red, green, blue and alpha values, all in range `[0, 1]` or all in range `[0, 255]` if any * value is > 1. * - * All values here are optional. Color channels default to 0, the alpha channel defaults to 1. + * All values here are optional. Colour channels default to 0, the alpha channel defaults to 1. */ @Serializable data class Colour( @@ -85,4 +91,26 @@ data class Colour( /** True if all values are between `[0..1]`. */ fun isPercentage(): Boolean = !isDecimal() + fun asDecimal() = if (isPercentage()) { + Colour( + (red * 255).coerceIn(0f..255f), + (green * 255).coerceIn(0f..255f), + (blue * 255).coerceIn(0f..255f), + (alpha * 255).coerceIn(0f..255f), + ) + } else { + this + } + + fun asPercentage() = if (isPercentage()) { + this + } else { + Colour( + (red / 255).coerceIn(0f..1f), + (green / 255).coerceIn(0f..1f), + (blue / 255).coerceIn(0f..1f), + (alpha / 255).coerceIn(0f..1f), + ) + } + } diff --git a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioPrototypes.kt b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioPrototypes.kt index 8cedb60..ea820a4 100644 --- a/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioPrototypes.kt +++ b/modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioPrototypes.kt @@ -36,7 +36,7 @@ object FactorioPrototypeSerializer : JsonContentPolymorphicSerializer FactorioMapTilePrototype.serializer() + FactorioPrototype.PrototypeObjectName.LuaTilePrototype -> MapTilePrototype.serializer() null -> throw Exception("Unknown FactorioPrototype $key: '$type' ") } @@ -45,10 +45,10 @@ object FactorioPrototypeSerializer : JsonContentPolymorphicSerializer, val order: String, /** Can the tile be mined for resources? */ diff --git a/modules/events-server/build.gradle.kts b/modules/events-server/build.gradle.kts index f1e7b8c..2495d2f 100644 --- a/modules/events-server/build.gradle.kts +++ b/modules/events-server/build.gradle.kts @@ -31,10 +31,11 @@ dependencies { implementation(platform(libs.http4k.bom)) implementation(libs.bundles.http4k) - implementation(libs.http4k.format.kotlinx) implementation(libs.bundles.logging) + + implementation("com.sksamuel.scrimage:scrimage-core:4.0.24") } application { diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/GlobalKTableDefinition.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/GlobalKTableDefinition.kt new file mode 100644 index 0000000..d2b3fb7 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/GlobalKTableDefinition.kt @@ -0,0 +1,31 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import org.apache.kafka.streams.state.QueryableStoreTypes.timestampedKeyValueStore + +/** + * * update topic [topicName] + * * plus table-source topic [tableName] + * * plus state store [globalStoreName] + * + * equals a GlobalTableDefinition + */ +abstract class GlobalKTableDefinition( + val tableName: String, + override val serdes: KeyValueSerdes, + val globalStoreName: String = "$tableName-store", + /** + * The backing topic, from which this table will be filled. + * + * Updates to items can be sent or received with this topic. + */ + final override val topicName: String = "$tableName-topic", + val storeType: TimestampedQueryStoreType = timestampedKeyValueStore(), +) : TopicDefinition { + val topicNames: Set + + init { + require(tableName != topicName) + require(tableName != globalStoreName) + topicNames = setOf(tableName, topicName) + } +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KStreamExtensions.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KStreamExtensions.kt new file mode 100644 index 0000000..28d1541 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KStreamExtensions.kt @@ -0,0 +1,111 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import dev.adamko.kafkatorio.extensions.kafkastreams.KStreamExtensions.flatMap +import dev.adamko.kafkatorio.extensions.kafkastreams.KStreamExtensions.groupBy +import dev.adamko.kafkatorio.extensions.kafkastreams.KStreamExtensions.map +import dev.adamko.kafkatorio.extensions.kafkastreams.KStreamExtensions.mapValues +import dev.adamko.kafkatorio.extensions.kafkastreams.KafkaStreamsExtensions.withName +import dev.adamko.kafkatorio.extensions.kafkastreams.KeyPairExtensions.kvPair +import dev.adamko.kafkatorio.extensions.kafkastreams.KeyPairExtensions.toKeyValue +import org.apache.kafka.streams.KeyValue +import org.apache.kafka.streams.kstream.Grouped +import org.apache.kafka.streams.kstream.KGroupedStream +import org.apache.kafka.streams.kstream.KStream +import org.apache.kafka.streams.kstream.Transformer +import org.apache.kafka.streams.processor.ProcessorContext + +object KStreamExtensions { + +// /** @see KStream.map */ +// inline fun > KStream.map( +// name: String, +// crossinline mapper: (key: inK, value: inV) -> outV +// ): KStream = +// map({ k, v -> mapper(k, v).asKeyValue() }, withName(name)) + + /** @see KStream.map */ + inline fun KStream.map( + name: String, + crossinline mapper: (key: inK, value: inV) -> Pair + ): KStream = + map({ k, v -> mapper(k, v).toKeyValue() }, withName(name)) + + /** Wrapper for [KStream.mapValues] */ + fun KStream.mapValues( + name: String, + mapper: (key: K, value: inV) -> outV + ): KStream = + mapValues({ k, v -> mapper(k, v) }, withName(name)) + +// /** @see KStream.flatMap */ +// inline fun > KStream.flatMap( +// name: String, +// crossinline mapper: (key: inK, value: inV) -> Iterable +// ): KStream = flatMap( +// { k, v -> mapper(k, v).map { a: outV -> a.asKeyValue() } }, +// withName(name) +// ) + + /** @see KStream.flatMap */ + inline fun KStream.flatMap( + name: String, + crossinline mapper: (key: inK, value: inV) -> Iterable> + ): KStream = flatMap( + { k, v -> mapper(k, v).map { it.toKeyValue() } }, + withName(name) + ) + + /** @see KStream.groupBy */ + inline fun KStream.groupBy( + grouped: Grouped, + crossinline keySelector: KeyValue.() -> outK + ): KGroupedStream = + groupBy( + { k, v -> (k kvPair v).keySelector() }, + grouped + ) + + + +// @JvmName("flatMapKeyValuePairs") +// inline fun KStream.flatMap( +// name: String, +// crossinline mapper: (key: inK, value: inV) -> Iterable> +// ): KStream = +// flatMap( +// { k, v -> mapper(k, v).map { it.asKeyPair() } }, +// withName(name) +// ) + +// inline fun KStream.mapValues( +// name: String, +// crossinline mapper: (readOnlyKey: K, value: inV) -> outV +// ): KStream = +// map({ k, v -> (k to mapper(v)).asKeyPair() }, KafkaStreamsExtensions.withName(name)) + +} + + +//open class TransformerKt( +//) : Transformer> { +// +// val onClose: TransformerKt.() -> Unit = {} +// val transformer: TransformerKt.(key: inK, value: inV) -> KeyValue? = +// {} +// +// private lateinit var _context: ProcessorContext +// val context: ProcessorContext? +// get() = if (this::_context.isInitialized) _context else null +// +// override fun init(context: ProcessorContext) { +// this._context = context +// } +// +// override fun close() { +// onClose() +// } +// +// override fun transform(key: inK, value: inV): KeyValue? { +// return transformer(key, value) +// } +//} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KStreamTransform.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KStreamTransform.kt new file mode 100644 index 0000000..b6cfd63 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KStreamTransform.kt @@ -0,0 +1,90 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import dev.adamko.kafkatorio.extensions.kafkastreams.KeyPairExtensions.toKeyValue +import kotlin.experimental.ExperimentalTypeInference +import org.apache.kafka.streams.KeyValue +import org.apache.kafka.streams.kstream.KStream +import org.apache.kafka.streams.kstream.Transformer +import org.apache.kafka.streams.kstream.TransformerSupplier +import org.apache.kafka.streams.processor.ProcessorContext + +/** @see KStream.transform */ +inline fun KStream.transformKt( + crossinline block: BuildTransformer +): KStream { + + val transformer = object : Transformer> { + + private lateinit var scope: TransformerScope + + override fun init(procContext: ProcessorContext) { + scope = TransformerScope(procContext) + scope.block() + } + + override fun close() { + scope.close() + } + + override fun transform(key: inK, value: inV): KeyValue? { + return scope.mapper(key, value)?.toKeyValue() + } + } + + return transform({ transformer }) +} + +/** @see KStream.transform */ +@OptIn(ExperimentalTypeInference::class) +inline fun > KStream.transformKt2( + @BuilderInference block: Transformer.() -> Unit +): KStream { + + val transformer = object : Transformer> { + + private lateinit var scope: TransformerScope + + override fun init(procContext: ProcessorContext) { + scope = TransformerScope(procContext) + } + + override fun close() { + scope.close() + + sequence { + + } + } + + override fun transform(key: inK, value: inV): KeyValue? { + return scope.mapper(key, value)?.toKeyValue() + } + } + + return transform(TransformerSupplier { transformer }) +} + +typealias BuildTransformer = TransformerScope.() -> Unit + +typealias TransformerMapper = (key: inK, value: inV) -> Pair? +typealias TransformerCloseHandler = (ProcessorContext) -> Unit + +open class TransformerScope( + val processorContext: ProcessorContext, +) : AutoCloseable { + + private var closeHandler: TransformerCloseHandler = {} + lateinit var mapper: TransformerMapper + + fun handleClose(closeHandler: TransformerCloseHandler) { + this.closeHandler = closeHandler + } + + fun mapEach(mapper: TransformerMapper) { + this.mapper = mapper + } + + override fun close() { + closeHandler(processorContext) + } +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KTableExtensions.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KTableExtensions.kt new file mode 100644 index 0000000..c573dd3 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KTableExtensions.kt @@ -0,0 +1,19 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import dev.adamko.kafkatorio.extensions.kafkastreams.KTableExtensions.mapValues +import dev.adamko.kafkatorio.extensions.kafkastreams.KafkaStreamsExtensions.withName +import org.apache.kafka.streams.kstream.KTable + +object KTableExtensions { + + /** @see [KTable.mapValues] */ + fun KTable.mapValues( + name: String, + mapper: (readOnlyKey: K, value: inV) -> outV + ): KTable = + mapValues( + { k, v -> mapper(k, v) }, + withName(name) + ) + +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KafkaStreamsExtensions.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KafkaStreamsExtensions.kt new file mode 100644 index 0000000..1bea403 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KafkaStreamsExtensions.kt @@ -0,0 +1,93 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import org.apache.kafka.common.serialization.Serde +import org.apache.kafka.streams.kstream.Grouped +import org.apache.kafka.streams.kstream.Joined +import org.apache.kafka.streams.kstream.Materialized +import org.apache.kafka.streams.kstream.Named +import org.apache.kafka.streams.kstream.Produced +import org.apache.kafka.streams.kstream.Repartitioned +import org.apache.kafka.streams.processor.StateStore + +/** Helpful extensions & shortcuts for Kafka Streams */ +object KafkaStreamsExtensions { + + /** @see [Named] */ + fun withName(name: String): Named = Named.`as`(name) + +// @Suppress("FunctionName") +// inline fun MaterializedAs( +// tableDefinition: GlobalTableDefinition +// ) = MaterializedAs( +// tableDefinition.globalStoreName, +// tableDefinition.serdes.keySerde, +// tableDefinition.serdes.valueSerde +// ) + + /** @see Materialized */ + inline fun materializedAs( + name: String, + keySerde: Serde,//= kxsJsonSerde(), + valueSerde: Serde,//= kxsJsonSerde(), + ): Materialized = + Materialized.`as`(name) + .withKeySerde(keySerde) + .withValueSerde(valueSerde) + + /** @see Repartitioned */ + inline fun repartitionedAs( + name: String, + keySerde: Serde, + valueSerde: Serde, + ): Repartitioned = + Repartitioned.`as`(name) + .withKeySerde(keySerde) + .withValueSerde(valueSerde) + + /** @see Produced */ + fun producedAs(name: String): Produced = + Produced.`as`(name) + + /** @see Produced */ + inline fun producedAs( + name: String, + keySerde: Serde, + valueSerde: Serde, + ): Produced = + producedAs(name) + .withKeySerde(keySerde) + .withValueSerde(valueSerde) + +// @Suppress("FunctionName") +// fun JoinedAs(name: String): Joined = +// Joined.`as`(name) + + /** @see Joined */ + inline fun joinedAs( + name: String, + keySerde: Serde,// = kxsJsonSerde(), + valueSerde: Serde,//= kxsJsonSerde(), + otherValueSerde: Serde?,//= kxsJsonSerde(), + ): Joined = + Joined.`as`(name) + .withKeySerde(keySerde) + .withValueSerde(valueSerde) + .withOtherValueSerde(otherValueSerde) + + /** @see [Grouped.`as`] */ + inline fun groupedAs( + name: String, + keySerde: Serde?, //= kxsJsonSerde(), + valueSerde: Serde?,// = kxsJsonSerde(), + ): Grouped = + Grouped.`as`(name) + .withKeySerde(keySerde) + .withValueSerde(valueSerde) + +// fun InteractiveQueryService.getQueryableStore( +// tableDef: GlobalTableDefinition +// ): Optional>> { +// return getQueryableStore(tableDef.globalStoreName, tableDef.storeType) +// } + +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KeyPairExtensions.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KeyPairExtensions.kt new file mode 100644 index 0000000..0bb1d21 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KeyPairExtensions.kt @@ -0,0 +1,35 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import org.apache.kafka.streams.KeyValue +import org.apache.kafka.streams.state.ValueAndTimestamp + +object KeyPairExtensions { + + inline fun Pair.toKeyValue(): KeyValue = + KeyValue.pair(first, second) + + infix fun K.kvPair(value: V): KeyValue = KeyValue(this, value) + + infix fun V.timestamped(timestamp: Long): ValueAndTimestamp = + ValueAndTimestamp.make(this, timestamp) + + + @JvmName("keyValueGetKey") + operator fun KeyValue.component1(): K = key + @JvmName("keyValueGetValue") + operator fun KeyValue.component2(): V = value + + + @JvmName("keyValueTimestampGetKey") + operator fun KeyValue>.component1(): K = key + @JvmName("keyValueTimestampGetValue") + operator fun KeyValue>.component2(): V = value.value() + @JvmName("keyValueTimestampGetTimestamp") + operator fun KeyValue>.component3(): Long = value.timestamp() + + + @JvmName("valueTimestampGetValue") + operator fun ValueAndTimestamp.component1(): V = value() + @JvmName("valueTimestampGetTimestamp") + operator fun ValueAndTimestamp.component2(): Long = timestamp() +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KeyValueSerdes.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KeyValueSerdes.kt new file mode 100644 index 0000000..431af7f --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KeyValueSerdes.kt @@ -0,0 +1,40 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import dev.adamko.kafkatorio.extensions.kafkastreams.KotlinxSerializationExtensions.kxsJsonSerde +import org.apache.kafka.common.serialization.Serde +import org.apache.kafka.common.serialization.Serdes +import org.apache.kafka.streams.kstream.Consumed +import org.apache.kafka.streams.kstream.Produced +import org.apache.kafka.streams.kstream.Repartitioned + +/** + * A wrapper for two [Serde]s - [one for the Key][keySerde], [another for the Value][valueSerde]. + */ +open class KeyValueSerdes( + val keySerde: Serde, + val valueSerde: Serde, +) { + + fun consumer(name: String): Consumed = + Consumed.with(keySerde, valueSerde).withName(name) + + fun producer(name: String): Produced = + Produced.with(keySerde, valueSerde).withName(name) + + fun repartitioned(name: String): Repartitioned = + Repartitioned.with(keySerde, valueSerde).withName(name) + + companion object { + inline fun kxsJson(): KeyValueSerdes = + KeyValueSerdes( + kxsJsonSerde(), + kxsJsonSerde(), + ) + + fun string(): KeyValueSerdes = + KeyValueSerdes( + Serdes.String(), + Serdes.String(), + ) + } +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KotlinxSerializationExtensions.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KotlinxSerializationExtensions.kt new file mode 100644 index 0000000..86acd86 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/KotlinxSerializationExtensions.kt @@ -0,0 +1,55 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import org.apache.kafka.common.serialization.Deserializer +import org.apache.kafka.common.serialization.Serde +import org.apache.kafka.common.serialization.Serializer + +object KotlinxSerializationExtensions { + + inline fun Json.kafkaSerializer() = Serializer { topic: String, data: T -> + runCatching { + encodeToString(data).encodeToByteArray() +// jsonMapper.encodeToString(data as T).encodeToByteArray() + }.getOrElse { e -> + println( + """ + Exception on encodeToString + Topic: $topic + topicData: $data + topicData as T: ${data as T} + """.trimIndent() + ) + e.printStackTrace() + throw e + } + } + + inline fun Json.kafkaDeserializer() = Deserializer { topic: String, data: ByteArray -> + runCatching { + decodeFromString(data.decodeToString()) + }.getOrElse { e -> + println( + """ + Exception on decodeFromString, + Topic: $topic + topicData: $data + topicData as T: ${data as T} + """.trimIndent() + ) + e.printStackTrace() + throw e + } + } + + + inline fun Json.serde() = object : Serde { + + override fun serializer(): Serializer = kafkaSerializer() + + override fun deserializer(): Deserializer = kafkaDeserializer() + } + +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/TopicData.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/TopicData.kt new file mode 100644 index 0000000..dd23620 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/TopicData.kt @@ -0,0 +1,17 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import org.apache.kafka.streams.KeyValue + +interface TopicData { + + val topicKey: K + + companion object { + fun > T.toPair(): Pair = topicKey to this + fun > T.toKeyValue(): KeyValue = KeyValue.pair(topicKey, this) + + operator fun > T.component1(): K = topicKey + operator fun > T.component2(): T = this + + } +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/TopicDefinition.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/TopicDefinition.kt new file mode 100644 index 0000000..49ce40f --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/TopicDefinition.kt @@ -0,0 +1,42 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams; + +/** + * A definition for all topics that are created in Kafka. + * + * These can be used to create KTopics, KTables, or GlobalKTables. + */ +interface TopicDefinition { + + /** unique kafka processor id */ + val pid: String + get() = this::class.simpleName!! + + /** + * Unique name for each [TopicDefinition] + * + * https://devshawn.com/blog/apache-kafka-topic-naming-conventions/ + * + * https://www.xeotek.com/topic-naming-conventions-how-do-i-name-my-topics-5-recommendations-with-examples/ + * + * Domain: warehouse + * Operation: stock-management + * Data type: + * - cdc=all instances of a specific object/thing/item, + * - cmd=command (mutation), + * - fct=fact (information (complete or partial) + * about an event at specific point in time) + * - sys=internal system topic - not meant for external use + * - tbl=a global table store (must be paired with another topic) + * Description: the data type + * + * Version: stored in message header + * + * ``` + * warehouse.stock-management.sts/cdc/cmd.warehouse-item + * ``` + */ + val topicName: String + + val serdes: KeyValueSerdes + +} diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/aliases.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/aliases.kt new file mode 100644 index 0000000..8461387 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/extensions/kafkastreams/aliases.kt @@ -0,0 +1,7 @@ +package dev.adamko.kafkatorio.extensions.kafkastreams + +import org.apache.kafka.streams.state.QueryableStoreType +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore +import org.apache.kafka.streams.state.ValueAndTimestamp + +typealias TimestampedQueryStoreType = QueryableStoreType>> diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt index 914a02b..c80da00 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt @@ -1,17 +1,44 @@ package dev.adamko.kafkatorio.processor +import com.sksamuel.scrimage.ImmutableImage +import com.sksamuel.scrimage.nio.PngWriter +import com.sksamuel.scrimage.pixels.Pixel +import com.sksamuel.scrimage.pixels.PixelTools import dev.adamko.kafkatorio.events.schema.FactorioConfigurationUpdate import dev.adamko.kafkatorio.events.schema.FactorioEvent +import dev.adamko.kafkatorio.events.schema.FactorioObjectData import dev.adamko.kafkatorio.events.schema.KafkatorioPacket +import dev.adamko.kafkatorio.events.schema.MapChunk +import dev.adamko.kafkatorio.events.schema.MapChunkPosition +import dev.adamko.kafkatorio.events.schema.MapTile +import dev.adamko.kafkatorio.events.schema.MapTilePosition +import dev.adamko.kafkatorio.events.schema.MapTilePrototype +import dev.adamko.kafkatorio.events.schema.MapTiles +import dev.adamko.kafkatorio.extensions.kafkastreams.KafkaStreamsExtensions.materializedAs +import dev.adamko.kafkatorio.extensions.kafkastreams.KeyPairExtensions.toKeyValue +import dev.adamko.kafkatorio.extensions.kafkastreams.KotlinxSerializationExtensions.serde +import dev.adamko.kafkatorio.extensions.kafkastreams.TopicData +import dev.adamko.kafkatorio.extensions.kafkastreams.TopicData.Companion.toKeyValue +import java.awt.image.BufferedImage +import java.io.File import java.time.Duration +import kotlin.math.roundToInt +import kotlinx.serialization.Serializable import kotlinx.serialization.decodeFromString import kotlinx.serialization.encodeToString import org.apache.kafka.common.serialization.Serdes import org.apache.kafka.streams.KafkaStreams +import org.apache.kafka.streams.StoreQueryParameters import org.apache.kafka.streams.StreamsBuilder import org.apache.kafka.streams.TopologyDescription import org.apache.kafka.streams.kstream.Consumed +import org.apache.kafka.streams.kstream.KStream +import org.apache.kafka.streams.kstream.KTable import org.apache.kafka.streams.kstream.Produced +import org.apache.kafka.streams.state.QueryableStoreTypes +import org.apache.kafka.streams.state.ReadOnlyKeyValueStore +import org.apache.kafka.streams.state.ValueAndTimestamp + class KafkatorioTopology( private val websocketServer: WebsocketServer, @@ -20,36 +47,25 @@ class KafkatorioTopology( private val sourceTopic = "factorio-server-log" + private lateinit var tilePrototypesStore: ReadOnlyKeyValueStore> fun build() { - builder.stream( - sourceTopic, - Consumed.with(Serdes.String(), Serdes.String()) - ) - .mapValues { readOnlyKey, value -> - println("Mapping $readOnlyKey:$value") - jsonMapper.decodeFromString(value) - } - .peek { _, value -> - websocketServer.sendMessage(jsonMapper.encodeToString(value)) - } - .to( - { _, value, _ -> -// println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}") - when (value) { - is FactorioEvent -> - "kafkatorio.${value.packetType.name}.${value.data.objectName.name}" - is FactorioConfigurationUpdate -> - "kafkatorio.${value.packetType.name}" - } - }, - Produced.with(Serdes.String(), KafkatorioPacketSerde) - ) + parseAndSplitServerLog() + sendPlayerUpdatesToWebSocket() +// createMapTileImages() + val tilePrototypesTable: KTable = tilePrototypesTable() val topology = builder.build() val streams = KafkaStreams(topology, appProps.kafkaConfig) + tilePrototypesStore = streams.store( + StoreQueryParameters.fromNameAndType( + tilePrototypesTable.queryableStoreName(), + QueryableStoreTypes.timestampedKeyValueStore() + ) + ) + streams.setUncaughtExceptionHandler(StreamsExceptionHandler()) Runtime.getRuntime().addShutdownHook(Thread { streams.close(Duration.ofSeconds(1)) }) @@ -62,4 +78,272 @@ class KafkatorioTopology( streams.start() } + private fun parseAndSplitServerLog() { + + builder.stream( + sourceTopic, + Consumed.with(Serdes.String(), Serdes.String()) + ) + .mapValues { readOnlyKey, value -> + println("Mapping $readOnlyKey:$value") + jsonMapper.decodeFromString(value) + } + .to( + { _, value, _ -> +// println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}") + when (value) { + is FactorioEvent -> + "kafkatorio.${value.packetType.name}.${value.data.objectName.name}" + is FactorioConfigurationUpdate -> + "kafkatorio.${value.packetType.name}.FactorioConfigurationUpdate" + } + }, + Produced.with(Serdes.String(), KafkatorioPacketSerde) + ) + } + + private fun sendPlayerUpdatesToWebSocket() { + builder.stream( + "kafkatorio.${KafkatorioPacket.PacketType.EVENT}.${FactorioObjectData.ObjectName.LuaPlayer}", + Consumed.with(Serdes.String(), KafkatorioPacketSerde) + ) + .foreach { _, value -> + websocketServer.sendMessage(jsonMapper.encodeToString(value)) + } + } + + private fun tilePrototypesTable(): KTable { + return builder.stream( + "kafkatorio.${KafkatorioPacket.PacketType.CONFIG}.FactorioConfigurationUpdate", + Consumed.with(Serdes.String(), jsonMapper.serde()) + ) + .flatMap { _, update: FactorioConfigurationUpdate -> + update + .prototypes + .filterIsInstance() + .map { proto -> proto.name to proto } + .map { it.toKeyValue() } + } + .toTable( + materializedAs( + "kafkatorio.prototypes.map-tiles", + jsonMapper.serde(), + jsonMapper.serde() + ) + ) + } + + private fun mapChunksStream() { + + val luaTilesUpdatesStream: KStream = builder.stream( + "kafkatorio.${KafkatorioPacket.PacketType.EVENT}.${FactorioObjectData.ObjectName.MapChunk}", + Consumed.with(Serdes.String(), jsonMapper.serde()) + ) + .filter { _, event: FactorioEvent -> event.data is MapTiles } + .mapValues { _, event: FactorioEvent -> (event.data as? MapTiles)!! } + + val chunkTilesUpdateStream: KStream = builder.stream( + "kafkatorio.${KafkatorioPacket.PacketType.EVENT}.${FactorioObjectData.ObjectName.LuaTiles}", + Consumed.with(Serdes.String(), jsonMapper.serde()) + ) + .filter { _, event: FactorioEvent -> event.data is MapChunk } + .mapValues { _, event: FactorioEvent -> (event.data as? MapChunk)!!.tiles } + + @Serializable + data class TileUpdateRecordKey( + val surfaceIndex: Int, + val tilePosition: MapTilePosition, + ) + + @Serializable + data class TileUpdateRecord( + val surfaceIndex: Int, + val tilePosition: MapTilePosition, + val tile: MapTile + ) : TopicData { + override val topicKey: TileUpdateRecordKey = + TileUpdateRecordKey(surfaceIndex, tilePosition) + } + + val allTileUpdatesStream: KStream = + luaTilesUpdatesStream + .merge(chunkTilesUpdateStream) + .flatMap { _, mapTiles: MapTiles -> + mapTiles.tiles.map { tile -> + TileUpdateRecord( + mapTiles.surfaceIndex, + tile.position, + tile + ).toKeyValue() + } + } + + @Serializable + data class MapChunkDataPosition( + val position: MapChunkPosition, + val surfaceIndex: Int, + ) + + @Serializable + data class MapChunkData( + val position: MapChunkPosition, + val surfaceIndex: Int, + val tiles: Set, + ) : TopicData { + override val topicKey = MapChunkDataPosition(position, surfaceIndex) + + constructor(key: MapChunkDataPosition, tiles: Set) : + this(key.position, key.surfaceIndex, tiles) + } + + val chunksTable = + allTileUpdatesStream + .groupBy { _: TileUpdateRecordKey, value: TileUpdateRecord -> + MapChunkDataPosition( + value.tilePosition.toMapChunkPosition(), + value.surfaceIndex, + ) + } + .aggregate( + { setOf() }, + ) { _: MapChunkDataPosition, v: TileUpdateRecord, tiles: Set -> + tiles + v.tile + } + .mapValues { chunkPosition: MapChunkDataPosition, tiles: Set -> + MapChunkData(chunkPosition, tiles) + } + + chunksTable.toStream() + .foreach { key: MapChunkDataPosition, value: MapChunkData -> + + + value.tiles + .map { tile -> + + val colour = tilePrototypesStore.get(tile.prototypeName) + .value() + .mapColour + .asDecimal() + + + val pixelColour = PixelTools.argb( + colour.alpha.roundToInt(), + colour.red.roundToInt(), + colour.green.roundToInt(), + colour.blue.roundToInt(), + ) + + Pixel( + tile.position.x, + tile.position.y, + pixelColour + ) + } + + val chunkImg = ImmutableImage.create(32, 32, BufferedImage.TYPE_INT_ARGB) + + val zoom = 1u + chunkImg.output( + PngWriter.MaxCompression, + File("${key.surfaceIndex}/$zoom/${key.position.x}/${key.position.y}.png") + ) + } + } + +// +// private fun createMapTileImages() { +// +// val mapTilesByPrototypeName: KStream = +// builder.stream( +// "kafkatorio.${KafkatorioPacket.PacketType.EVENT}.${FactorioObjectData.ObjectName.LuaTiles}", +// Consumed.with(Serdes.String(), KafkatorioPacketSerde) +// ) +// .filter { _, packet -> packet is FactorioEvent && packet.data is MapTiles } +// .mapValues { packet -> ((packet as? FactorioEvent)?.data as? MapTiles)!! } +// .flatMap { _, mapTiles -> +// mapTiles +// .tiles +// .map { tile -> tile.prototypeName to tile } +// .map { it.toKeyValue() } +// } +//// .groupByKey() +// +// @Serializable +// data class TileSurfaceCoord( +// val surfaceIndex: Int, +// val position: MapTilePosition, +// ) +// +// @Serializable +// data class TileSurfaceState( +// val surfaceIndex: Int, +// val position: MapTilePosition, +// val prototypeName: String, +// ) +// +// val mapTilesTable: KTable = +// builder.stream( +// "kafkatorio.${KafkatorioPacket.PacketType.EVENT}.${FactorioObjectData.ObjectName.LuaTiles}", +// Consumed.with(Serdes.String(), KafkatorioPacketSerde) +// ) +// .filter { _, packet -> packet is FactorioEvent && packet.data is MapTiles } +// .mapValues { packet -> ((packet as? FactorioEvent)?.data as? MapTiles)!! } +// .flatMap({ _, mapTiles: MapTiles -> +// mapTiles.tiles.map { tile: MapTile -> +// val key = TileSurfaceCoord(mapTiles.surfaceIndex, tile.position) +// val value = TileSurfaceState(mapTiles.surfaceIndex, tile.position, tile.prototypeName) +// (key to value).toKeyValue() +// } +// } +// ) +// .toTable( +// materializedAs( +// "kafkatorio.table.map-tile-status", +// jsonMapper.serde(), +// jsonMapper.serde(), +// ) +// ) +// +// +// val tilePrototypesTable: KTable = +// builder.stream( +// "kafkatorio.${KafkatorioPacket.PacketType.CONFIG}.FactorioConfigurationUpdate", +// Consumed.with(Serdes.String(), KafkatorioPacketSerde) +// ) +// .flatMap(KeyValueMapper>> { _, packet -> +// when (packet) { +// is FactorioConfigurationUpdate -> +// packet.prototypes +// .filterIsInstance() +// .map { proto -> proto.name to proto } +// .map { it.toKeyValue() } +// is FactorioEvent -> +// emptyList() +// } +// }) +// .toTable( +// materializedAs("kafkatorio.table.tile-prototypes", Serdes.String(), jsonMapper.serde()) +// ) +// +// +// @Serializable +// data class TileSurfaceData( +// val surfaceIndex: Int, +// val position: MapTilePosition, +// val prototype: MapTilePrototype, +// ) +// +// val enrichedTiles: KTable = +// mapTilesTable.join( +// tilePrototypesTable, +// { tile: TileSurfaceState -> +// tile.prototypeName +// }, +// { tile: TileSurfaceState, proto: MapTilePrototype -> +// TileSurfaceData(tile.surfaceIndex, tile.position, proto) +// }, +// ) +// +// } + } diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes.kt index 88c3666..e178d78 100644 --- a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes.kt +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes.kt @@ -12,7 +12,6 @@ import org.http4k.format.ConfigurableKotlinxSerialization import org.http4k.format.asConfigurable import org.http4k.format.withStandardMappings - object KafkatorioPacketSerde : Serde { override fun serializer() = Serializer { _, message -> jsonMapper.encodeToString(message).encodeToByteArray() @@ -23,7 +22,7 @@ object KafkatorioPacketSerde : Serde { } } -val jsonMapper = Json { +val jsonMapper: Json = Json { prettyPrint = true prettyPrintIndent = " " serializersModule = jsonMapperKafkatorio.serializersModule diff --git a/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/FactorioGameState.kt b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/FactorioGameState.kt new file mode 100644 index 0000000..d66e7d1 --- /dev/null +++ b/modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/FactorioGameState.kt @@ -0,0 +1,68 @@ +package dev.adamko.kafkatorio.processor.topology + +import dev.adamko.kafkatorio.events.schema.KafkatorioPacket +import dev.adamko.kafkatorio.events.schema.MapChunkPosition +import dev.adamko.kafkatorio.events.schema.MapTilePosition +import dev.adamko.kafkatorio.events.schema.MapTilePrototype +import dev.adamko.kafkatorio.processor.jsonMapper +import kotlinx.serialization.Serializable +import kotlinx.serialization.decodeFromString +import org.apache.kafka.common.serialization.Serdes +import org.apache.kafka.streams.StreamsBuilder +import org.apache.kafka.streams.kstream.Consumed +import org.apache.kafka.streams.kstream.KGroupedStream + +class FactorioGameState ( +private val builder: StreamsBuilder = StreamsBuilder(), +) { + private val sourceTopic = "factorio-server-log" + + + private fun gatherGameState() { + + val serverPackets: KGroupedStream = + builder.stream( + sourceTopic, + Consumed.with(Serdes.String(), Serdes.String()) + ) + .mapValues { readOnlyKey, value -> + println("Mapping $readOnlyKey:$value") + jsonMapper.decodeFromString(value) + } + .groupByKey() + + @Serializable + data class TileSurfaceData( + val surfaceIndex: Int, + val position: MapTilePosition, + val prototype: MapTilePrototype, + ) + + @Serializable + data class MapChunkData( + val position: MapChunkPosition, + val tiles: List, + ) + + @Serializable + data class FactorioSurfaceData( + val surfaceIndex: Int, + val chunks: List, + ) + + @Serializable + data class FactorioServerState( + val surfaces: Map, + val tilePrototypes: List, + ) + + + +// serverPackets +// .aggregate( +// Initializer { FactorioServerState("", emptyMap(), emptyList()) } +// ) + } + + +}