lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

- serialization improvements (the tests work at least)

Browse source 
- fixing some version mis-alignments
This commit is contained in:
Adam 2021-12-01 13:35:32 +01:00
parent babe0161ae
commit 53862ef1b5
19 changed files with 305 additions and 260 deletions
Download patch file Download diff file Expand all files Collapse all files

6
modules/factorio-events-data-model/build.gradle.kts
View file

@ -10,9 +10,9 @@ val projectId: String by project.extra
val buildDir: Directory = layout.buildDirectory.dir(projectId).get()
dependencies {
val kotlinXSerializationVersion = "1.3.0"
implementation("org.jetbrains.kotlinx:kotlinx-serialization-core:$kotlinXSerializationVersion")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:$kotlinXSerializationVersion")
implementation(platform(libs.kotlinx.serialization.bom))
implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json)
implementation("com.github.ntrrgc:ts-generator:1.1.2")
implementation("org.jetbrains.kotlin:kotlin-reflect")

78
modules/factorio-events-data-model/src/main/kotlin/dev/adamko/factorioevents/model/factorioEvents.kt
View file

@ -1,61 +1,101 @@
package dev.adamko.factorioevents.model
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonClassDiscriminator
//@Serializable(with = FactorioEventSerializer::class)
@Serializable
data class FactorioEvent<T : FactorioObjectData>(
data class FactorioEvent<out T : FactorioObjectData>(
/** the initial Factorio event ({defines.events}) trigger */
@SerialName("event_type")
val eventType: String,
/** Schema versioning */
@SerialName("mod_version")
val modVersion: String,
/** game time */
@SerialName("tick")
val tick: UInt,
val data: T,
@SerialName("data")
val data: T
)
@JsonClassDiscriminator("object_name")
sealed interface FactorioObjectData {
val objectName: String
/** The [JsonClassDiscriminator] for [FactorioEvent] */
const val FactorioObjectDataDiscriminatorKey: String = "object_name"
@Serializable
@JsonClassDiscriminator(FactorioObjectDataDiscriminatorKey)
sealed class FactorioObjectData {
@SerialName(FactorioObjectDataDiscriminatorKey)
abstract val objectName: String
}
@Serializable
@SerialName("LuaPlayer")
data class PlayerData(
@SerialName(FactorioObjectDataDiscriminatorKey)
override val objectName: String,
val associatedCharactersUnitNumbers: List<UInt>,
val characterUnitNumber: UInt?,
val name: String,
val position: PositionData,
) : FactorioObjectData
@Serializable(with = ListAsObjectSerializer::class)
@SerialName("associated_characters_unit_numbers")
val associatedCharactersUnitNumbers: List<UInt>,
@SerialName("character_unit_number")
val characterUnitNumber: UInt?,
@SerialName("name")
val name: String,
@SerialName("position")
val position: PositionData,
) : FactorioObjectData()
@Serializable
@SerialName("LuaEntity")
data class EntityData(
@SerialName(FactorioObjectDataDiscriminatorKey)
override val objectName: String,
@SerialName("active")
val active: Boolean,
val health: Float?,
val healthRatio: Float,
@SerialName("health")
val health: Double?,
@SerialName("health_ratio")
val healthRatio: Double,
@SerialName("name")
val name: String,
@SerialName("position")
val position: PositionData,
@SerialName("surface_index")
val surfaceIndex: Int,
@SerialName("type")
val type: String,
@SerialName("unit_number")
val unitNumber: UInt?,
val playerIndex: UInt?,
) : FactorioObjectData
@SerialName("player_index")
val playerIndex: UInt? = null,
) : FactorioObjectData()
@Serializable
@SerialName("LuaSurface")
data class SurfaceData(
@SerialName(FactorioObjectDataDiscriminatorKey)
override val objectName: String,
val daytime: Float,
@SerialName("daytime")
val daytime: Double,
@SerialName("index")
val index: UInt,
@SerialName("name")
val name: String,
) : FactorioObjectData
) : FactorioObjectData()
@Serializable
data class PositionData(
val x: Int,
val y: Int,
)
@SerialName("x")
val x: Double,
@SerialName("y")
val y: Double,
)

34
modules/factorio-events-data-model/src/main/kotlin/dev/adamko/factorioevents/model/json.kt Normal file
View file

@ -0,0 +1,34 @@
package dev.adamko.factorioevents.model
import kotlinx.serialization.KSerializer
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonTransformingSerializer
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.modules.SerializersModule
/** Factorio outputs lists as Json objects - this serializer converts an object back to a list. */
class ListAsObjectSerializer<T>(dataSerializer: KSerializer<T>) :
JsonTransformingSerializer<List<T>>(ListSerializer(dataSerializer)) {
override fun transformDeserialize(element: JsonElement): JsonElement =
JsonArray(
element.jsonObject.values.toList()
)
override fun transformSerialize(element: JsonElement): JsonElement =
JsonObject(
element.jsonArray.mapIndexed { index, jsonElement -> "$index" to jsonElement }.toMap()
)
}
val jsonMapper = Json {
prettyPrint = true
prettyPrintIndent = " "
serializersModule = SerializersModule { }
}

139
modules/factorio-events-data-model/src/test/kotlin/dev/adamko/factorioevents/model/FactorioEventTest.kt Normal file
View file

@ -0,0 +1,139 @@
package dev.adamko.factorioevents.model
import io.kotest.core.spec.style.BehaviorSpec
import io.kotest.matchers.equality.shouldBeEqualToComparingFields
import kotlinx.serialization.decodeFromString
import org.intellij.lang.annotations.Language
class FactorioEventTest : BehaviorSpec({
Given("json on_player_joined_game") {
@Language("JSON")
val json = """
{
"data": {
"object_name": "LuaPlayer",
"name": "fredthedeadhead",
"character_unit_number": 1,
"associated_characters_unit_numbers": {},
"position": {
"x": -3.5703125,
"y": 29.75
}
},
"event_type": "on_player_joined_game",
"mod_version": "0.0.4",
"tick": 1278458
}
""".trimIndent()
Then("parse") {
val actual: FactorioEvent<FactorioObjectData> = jsonMapper.decodeFromString(json)
val expected = FactorioEvent(
data = PlayerData(
objectName = "LuaPlayer",
name = "fredthedeadhead",
characterUnitNumber = 1u,
associatedCharactersUnitNumbers = emptyList(),
position = PositionData(
x = -3.5703125,
y = 29.75
)
),
eventType = "on_player_joined_game",
modVersion = "0.0.4",
tick = 1278458u
)
actual shouldBeEqualToComparingFields expected
}
}
Given("json surface") {
@Language("JSON")
val json = """
{
"data": {
"object_name": "LuaSurface",
"name": "nauvis",
"index": 1,
"daytime": 0.8390800000511436707029133685864508152008056640625
},
"event_type": "on_tick",
"mod_version": "0.0.4",
"tick": 1278480
}
""".trimIndent()
Then("parse") {
val actual: FactorioEvent<FactorioObjectData> = jsonMapper.decodeFromString(json)
val expected = FactorioEvent(
data = SurfaceData(
objectName = "LuaSurface",
name = "nauvis",
index = 1u,
daytime = 0.8390800000511436707029133685864508152008056640625
),
eventType = "on_tick",
modVersion = "0.0.4",
tick = 1278480u
)
actual shouldBeEqualToComparingFields expected
}
}
Given("json entity") {
@Language("JSON")
val json = """
{
"data": {
"object_name": "LuaEntity",
"name": "character",
"type": "character",
"active": true,
"health": 250,
"health_ratio": 1,
"surface_index": 1,
"unit_number": 1,
"position": {
"x": -0.25,
"y": 30.09375
}
},
"event_type": "on_player_changed_position",
"mod_version": "0.0.6",
"tick": 1384507
}
""".trimIndent()
Then("parse") {
val actual: FactorioEvent<FactorioObjectData> = jsonMapper.decodeFromString(json)
val expected = FactorioEvent(
data = EntityData(
objectName = "LuaEntity",
name = "character",
type = "character",
active = true,
health = 250.0,
healthRatio = 1.0,
surfaceIndex = 1,
unitNumber = 1u,
playerIndex = null,
position = PositionData(
x = -0.25,
y = 30.09375,
)
),
eventType = "on_player_changed_position",
modVersion = "0.0.6",
tick = 1384507u
)
actual shouldBeEqualToComparingFields expected
}
}
})

21
modules/factorio-events-processor/build.gradle.kts
View file

@ -4,8 +4,7 @@ plugins {
application
}
description =
"""
description = """
Receive raw Factorio Events from Kafka and process them into targeted topics or data formats.
Factorio events -> Mod -> Kafka -> ***Factorio Events Processor*** -> processed events
@ -19,13 +18,23 @@ dependencies {
implementation("org.apache.kafka:kafka-streams:3.0.0")
implementation(libs.kotlinx.serialization)
implementation(projects.modules.factorioEventsDataModel)
// implementation("com.ionspin.kotlin:bignum:0.3.3")
implementation(enforcedPlatform(libs.kotlinx.serialization.bom))
implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json)
implementation(platform(libs.http4k.bom))
implementation(libs.http4k.core)
// implementation(libs.http4k.core)
implementation(libs.http4k.cloudnative)
implementation(libs.http4k.format.json)
implementation(libs.http4k.format.yaml)
// implementation(libs.http4k.format.json)
implementation(libs.http4k.format.yaml) {
because("parsing yaml properties files")
}
implementation(libs.http4k.format.kotlinx)
implementation(libs.bundles.logging)

1
modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/ApplicationProperties.kt
View file

@ -9,7 +9,6 @@ import org.http4k.cloudnative.env.Environment
import org.http4k.cloudnative.env.EnvironmentKey
import org.http4k.core.Credentials
import org.http4k.core.Uri
import org.http4k.format.Jackson
import org.http4k.format.Json
import org.http4k.lens.boolean
import org.http4k.lens.map

8
modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/main.kt
View file

@ -1,7 +1,8 @@
package dev.adamko.factorioevents.processor
import dev.adamko.factorioevents.model.FactorioEvent
import dev.adamko.factorioevents.model.jsonMapper
import dev.adamko.factorioevents.processor.config.ApplicationProperties
import dev.adamko.factorioevents.processor.model.FactorioServerLogRecord
import java.time.Duration
import org.apache.kafka.common.serialization.Serdes
import org.apache.kafka.streams.KafkaStreams
@ -9,7 +10,7 @@ import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.TopologyDescription
import org.apache.kafka.streams.kstream.Consumed
import org.apache.kafka.streams.processor.TopicNameExtractor
import org.http4k.format.Jackson
import org.http4k.format.KotlinxSerialization
fun main() {
@ -23,7 +24,8 @@ fun main() {
)
.mapValues { readOnlyKey, value ->
println("Mapping $readOnlyKey:$value")
Jackson.asA<FactorioServerLogRecord<*>>(value)
// jsonMapper.decodeFromString<FactorioEvent<*>>(value)
KotlinxSerialization.asA<FactorioEvent<*>>(value)
}
.to(TopicNameExtractor { key, value, recordContext ->
value.data.objectName

61
modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/model/FactorioServerLogRecord.kt
View file

@ -1,61 +0,0 @@
package dev.adamko.factorioevents.processor.model
import com.fasterxml.jackson.annotation.JsonProperty
import com.fasterxml.jackson.annotation.JsonTypeInfo
import com.fasterxml.jackson.annotation.JsonTypeName
import com.fasterxml.jackson.annotation.JsonValue
import java.math.BigDecimal
import java.math.BigInteger
import kotlin.reflect.KClass
data class FactorioServerLogRecord<DataType : FactorioObjectData>(
@JsonProperty("mod_version")
val modVersion: String,
@JsonProperty("tick")
val tick: BigInteger,
@JsonProperty("event_type")
val eventType: String,
@JsonProperty("data")
val data: DataType,
)
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.EXISTING_PROPERTY,
property = "object_name",
defaultImpl = UnknownFactorioObject::class,
visible = true,
)
sealed class FactorioObjectData(
@JsonProperty("object_name")
val objectName: String
)
object UnknownFactorioObject : FactorioObjectData("unknown")
@JsonTypeName("LuaSurface")
class LuaSurfaceData(
@JsonProperty("name")
val name: String,
@JsonProperty("index")
val index: Int,
@JsonProperty("daytime")
val daytime: BigDecimal,
) : FactorioObjectData("LuaSurface")
/*
{
"tick": 1144860,
"object_name": "LuaSurface",
"event_type": "on_tick",
"data": {
"name": "nauvis",
"index": 1,
"daytime": 0.4942800000457983
}
}
*/

68
modules/factorio-events-processor/src/test/kotlin/dev/adamko/factorioevents/processor/model/FactorioServerLogRecordTest.kt
View file

@ -1,68 +0,0 @@
package dev.adamko.factorioevents.processor.model
import io.kotest.assertions.json.shouldEqualJson
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe
import io.kotest.matchers.shouldNotBe
import org.http4k.format.Jackson
import org.intellij.lang.annotations.Language
class FactorioServerLogRecordTest : FunSpec({
test("player obj") {
@Language("JSON")
val example = """
{
"mod_version":"0.0.3",
"tick":1256529,
"event_type":"on_player_joined_game",
"data":{
"object_name":"LuaPlayer",
"name":"fredthedeadhead",
"character_unit_number":1,
"associated_characters_unit_numbers":{
},
"position":{
"x":3.48828125,
"y":23.43359375
}
}
}
""".trimIndent()
val record: FactorioServerLogRecord<*> = Jackson.asA(example)
record shouldBe null
}
test("surface obj") {
@Language("JSON")
val example = """
{
"mod_version":"0.0.3",
"tick":1256529,
"event_type":"on_player_joined_game",
"data":{
"object_name":"LuaEntity",
"name":"character",
"type":"character",
"active":true,
"health":250,
"surface_index":1,
"unit_number":1,
"position":{
"x":3.48828125,
"y":23.43359375
}
}
}
""".trimIndent()
val record: FactorioServerLogRecord<LuaSurfaceData> = Jackson.asA(example)
record shouldNotBe null
val str = Jackson.asFormatString(record)
str.shouldEqualJson(example)
}
})

2
modules/kt-rcon/build.gradle.kts
View file

@ -8,6 +8,8 @@ plugins {
dependencies {
implementation(enforcedPlatform(libs.kotlinx.serialization.bom))
implementation(libs.kotlinx.coroutines)
implementation("io.ktor:ktor-network:1.6.4")

2
modules/mod/build.gradle.kts
View file

@ -309,7 +309,7 @@ tasks.register("downloadFactorioApiDocs") {
)
val json = downloadedFile.readText()
val prettyJson = groovy.json.JsonOutput.prettyPrint(json)
val prettyJson = JsonOutput.prettyPrint(json)
apiFile.asFile.writeText(prettyJson)

10
modules/mod/src/main/typescript/generated/data-model.ts
View file

@ -13,8 +13,8 @@ interface FactorioEvent<T extends FactorioObjectData> {
}
interface PositionData {
x: int;
y: int;
x: double;
y: double;
}
interface PlayerData extends FactorioObjectData {
@ -27,8 +27,8 @@ interface PlayerData extends FactorioObjectData {
interface EntityData extends FactorioObjectData {
active: boolean;
health: float | null;
health_ratio: float;
health: double | null;
health_ratio: double;
name: string;
object_name: string;
player_index: uint | null;
@ -39,7 +39,7 @@ interface EntityData extends FactorioObjectData {
}
interface SurfaceData extends FactorioObjectData {
daytime: float;
daytime: double;
index: uint;
name: string;
object_name: string;

60
modules/mod/src/main/typescript/model/model.ts
View file

@ -1,60 +0,0 @@
// import {Serdes} from "../serdes/serdes";
// import FactorioObjectData = Serdes.FactorioObjectData;
//
// export type JsonPrimitiveType = number | string | boolean | undefined | null | void
//
// export type JsonArray<T extends JsonPrimitiveType = void> = T[]
//
// export type JsonTableValueType =
// JsonPrimitiveType
// | JsonTable
// | JsonArray<JsonPrimitiveType>
// | undefined
// | null
//
// export interface JsonTable {
// [key: string]: JsonTableValueType
// }
// export interface FactorioEvent extends JsonTable {
// /** Schema versioning */
// mod_version: string,
// /** game time */
// tick: uint,
// /** the initial Factorio event ({defines.events}) trigger */
// event_type: string,
// data: FactorioObjectData,
// }
//
// export const FactorioEvent =
// (<DataType extends FactorioObjectData>(
// tick: uint,
// eventType: string,
// data: DataType
// ) => {
// return {
// mod_version: MOD_VERSION,
// tick: tick,
// event_type: eventType,
// data: data,
// } as FactorioEvent
// }
// )
// export type JsonArray<TValue extends JsonPrimitiveType> = TValue[]
// export class JsonArray extends Array<JsonPrimitiveTypes> {
// }
// export type JsonArray = Array<JsonPrimitiveTypes>
// declare type JsonArrayConstructor = (new () => JsonArray)
// declare const JsonArray: JsonArrayConstructor
// export class JsonTable extends LuaTable<string, JsonTableValueType> {
// }
// export type JsonTable = LuaTable<string, JsonTableValueType>
// const JsonTable = () => new LuaTable<string, JsonTableValueType>()
// export type JsonTableConstructor = (new () => JsonTable)
// declare const JsonTable: JsonTableConstructor

8
modules/mod/src/main/typescript/package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "factorio-web-map-mod",
"version": "0.0.4",
"version": "0.0.6",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -83,9 +83,9 @@
}
},
"typescript": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.5.2.tgz",
"integrity": "sha512-5BlMof9H1yGt0P8/WF+wPNw6GfctgGjXp5hkblpyT+8rkASSmkUKMXrxR0Xg8ThVCi/JnHQiKXeBaEwCeQwMFw=="
"version": "4.4.4",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
"integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA=="
},
"typescript-to-lua": {
"version": "1.1.1",

4
modules/mod/src/main/typescript/package.json
View file

@ -1,6 +1,6 @@
{
"name": "factorio-web-map-mod",
"version": "0.0.5",
"version": "0.0.7",
"private": true,
"scripts": {
"build": "tstl",
@ -16,7 +16,7 @@
"@types/node": "^12.0.8",
"typescript-to-lua": "1.1.1",
"typed-factorio": "0.12.0",
"typescript": "4.5.2"
"typescript": "4.4.4"
},
"//": {
"typed-factorio": "0.10.0"

13
modules/mod/src/main/typescript/serdes/serdes.ts
View file

@ -1,18 +1,12 @@
// import {JsonTable, JsonArray} from "../model/model";
export namespace Serdes {
// export interface FactorioObjectData extends JsonTable {
// object_name: string
// }
export namespace Player {
export function playerToTable(player: LuaPlayer): PlayerData {
let charIds = Entity.entitiesToUnitNumbers(player.get_associated_characters())
return {
object_name: player.object_name,
name: player.name,
character_unit_number: player.character?.unit_number ?? null,
associated_characters_unit_numbers: charIds,
@ -30,6 +24,8 @@ export namespace Serdes {
return {
object_name: entity.object_name,
// entity data
name: entity.name,
type: entity.type,
active: entity.active,
@ -39,7 +35,7 @@ export namespace Serdes {
unit_number: entity.unit_number ?? null,
position: DataClasses.positionTableToTable(entity.position),
// player data:
// player data
player_index: player?.index ?? null
}
}
@ -61,6 +57,7 @@ export namespace Serdes {
export function surfaceToTable(surface: LuaSurface): SurfaceData {
return {
object_name: surface.object_name,
name: surface.name,
index: surface.index,
daytime: surface.daytime,