lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

packet size improvements

Browse source 
- change structure resource packets
- make more booleans default to false
- use encode_string - massive decrease in packet size!!
This commit is contained in:
Adam 2022-07-17 23:54:53 +02:00
parent 523f6d5914
commit 085776d42b
14 changed files with 131 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

97
modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/entities.kt
View file

@ -18,41 +18,23 @@ sealed interface FactorioEntityData {
/** Non-specific entity data */
@Serializable
@SerialName("kafkatorio.entity.FactorioEntityData")
@SerialName("kafkatorio.entity.FactorioEntityDataStandard")
data class Standard(
override val protoId: PrototypeId,
val status: Status? = null,
val unitNumber: UnitNumber? = null,
override val position: MapEntityPosition,
val graphicsVariation: UByte? = null,
val health: Float? = null,
val isActive: Boolean? = null,
val isRotatable: Boolean? = null,
val isActive: Boolean = false,
val isRotatable: Boolean = false,
val lastUser: UInt? = null,
) : FactorioEntityData {
constructor(
position: MapEntityPosition,
protoId: PrototypeId,
element: FactorioEntityUpdateElement.Entity
) : this(
protoId = protoId,
position = position,
status = element.status,
unitNumber = element.unitNumber,
graphicsVariation = element.graphicsVariation,
health = element.health,
isActive = element.isActive,
isRotatable = element.isRotatable,
lastUser = element.lastUser
)
}
val status: Status? = null,
val unitNumber: UnitNumber? = null,
) : FactorioEntityData
@Serializable
@SerialName("kafkatorio.entity.FactorioResourceData")
@SerialName("kafkatorio.entity.FactorioEntityDataResource")
data class Resource(
override val protoId: PrototypeId,
@ -60,19 +42,7 @@ sealed interface FactorioEntityData {
val amount: UInt,
/** Count of initial resource units contained. `null` if resource is not infinite. */
val initialAmount: UInt? = null,
) : FactorioEntityData {
constructor(
position: MapEntityPosition,
protoId: PrototypeId,
element: FactorioEntityUpdateElement.Resource
) : this(
protoId = protoId,
position = position,
amount = element.amount,
initialAmount = element.initialAmount,
)
}
) : FactorioEntityData
/**
@ -193,7 +163,7 @@ sealed interface FactorioEntityData {
}
sealed interface FactorioEntityUpdateDictionary<T : FactorioEntityUpdateElement>
sealed interface FactorioEntityUpdateDictionary<T : Any>
: Iterable<Pair<MapEntityPosition, T>> {
val entitiesXY: Map<String, Map<String, T>>
@ -203,28 +173,37 @@ sealed interface FactorioEntityUpdateDictionary<T : FactorioEntityUpdateElement>
override fun iterator(): Iterator<Pair<MapEntityPosition, T>> =
iterator {
entitiesXY.forEach { (x, column) ->
column.forEach { (y, entity) ->
yield(
MapEntityPosition(x.toDouble(), y.toDouble()) to entity
)
val invalid = mutableListOf<String>()
entitiesXY.forEach { (xString, column) ->
val x = xString.toDoubleOrNull()
column.forEach { (yString, entity) ->
val y = yString.toDoubleOrNull()
if (x == null || y == null) {
invalid += "$xString/$yString"
} else {
yield(MapEntityPosition(x, y) to entity)
}
}
}
if (invalid.isNotEmpty()) {
println("WARNING FactorioEntityUpdateDictionary contained invalid x/y coords: ${invalid.joinToString()}")
}
}
@Serializable
@SerialName("kafkatorio.entity.FactorioEntityUpdateResourceDictionary")
@SerialName("kafkatorio.entity.FactorioEntityUpdateResourceAmountDictionary")
@JvmInline
value class Resource(
override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Resource>>
) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Resource>
value class ResourceAmounts(
override val entitiesXY: Map<String, Map<String, UInt>> = emptyMap()
) : FactorioEntityUpdateDictionary<UInt>
@Serializable
@SerialName("kafkatorio.entity.FactorioEntityUpdateEntityDictionary")
@JvmInline
value class Entity(
override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Entity>>
) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Entity>
override val entitiesXY: Map<String, Map<String, FactorioEntityUpdateElement.Standard>>
) : FactorioEntityUpdateDictionary<FactorioEntityUpdateElement.Standard>
}
@ -232,24 +211,16 @@ sealed interface FactorioEntityUpdateDictionary<T : FactorioEntityUpdateElement>
sealed interface FactorioEntityUpdateElement {
@Serializable
@SerialName("kafkatorio.entity.FactorioEntityUpdateResource")
data class Resource(
val amount: UInt,
/** Count of initial resource units contained. `null` if resource is not infinite. */
val initialAmount: UInt? = null,
) : FactorioEntityUpdateElement
@Serializable
@SerialName("kafkatorio.entity.FactorioEntityUpdateEntity")
data class Entity(
@SerialName("kafkatorio.entity.FactorioEntityUpdateElementStandard")
data class Standard(
val status: FactorioEntityData.Status? = null,
val unitNumber: UnitNumber? = null,
val graphicsVariation: UByte? = null,
val health: Float? = null,
val isActive: Boolean? = null,
val isRotatable: Boolean? = null,
val isActive: Boolean = false,
val isRotatable: Boolean = false,
val lastUser: UInt? = null,
) : FactorioEntityUpdateElement

4
modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioPrototypes.kt
View file

@ -24,7 +24,7 @@ sealed interface FactorioPrototype {
val collisionMasks: LuaJsonList<String>,
val order: String,
/** Can the tile be mined for resources? */
val canBeMined: Boolean,
val canBeMined: Boolean = false,
) : FactorioPrototype
@ -63,7 +63,7 @@ sealed interface FactorioPrototype {
@Serializable
@SerialName("kafkatorio.prototype.EntityMiningProperties")
data class MiningProperties(
val canBeMined: Boolean,
val canBeMined: Boolean = false,
// @Contextual
val products: LuaJsonList<MinedProduct>?,
)

4
modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketInstant.kt
View file

@ -23,8 +23,8 @@ data class ConfigurationUpdate(
val factorioData: ConfigurationUpdateGameData,
// @Contextual
val allMods: LuaJsonList<ConfigurationUpdateModData>,
val modStartupSettingsChange: Boolean,
val migrationApplied: Boolean,
val modStartupSettingsChange: Boolean = false,
val migrationApplied: Boolean = false,
) : KafkatorioInstantPacketData() {
@Serializable

23
modules/events-library/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacketKeyed.kt
View file

@ -7,6 +7,7 @@ import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
import dev.adamko.kafkatorio.schema.common.EventName
import dev.adamko.kafkatorio.schema.common.FactorioEntityData
import dev.adamko.kafkatorio.schema.common.FactorioEntityUpdateDictionary
import dev.adamko.kafkatorio.schema.common.FactorioEntityUpdateDictionary.ResourceAmounts
import dev.adamko.kafkatorio.schema.common.ForceIndex
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapEntityPosition
@ -135,7 +136,7 @@ data class PlayerUpdate(
val kickedReason: String? = null,
val disconnectReason: String? = null,
/** `true` when a player is removed (deleted) from the game */
val isRemoved: Boolean? = null,
val isRemoved: Boolean = false,
) : KafkatorioKeyedPacketData()
@ -166,7 +167,14 @@ data class MapChunkEntityUpdate(
FactorioEntityData.Standard(
protoId = key.protoId,
position = position,
element = entity,
status = entity.status,
unitNumber = entity.unitNumber,
graphicsVariation = entity.graphicsVariation,
health = entity.health,
isActive = entity.isActive,
isRotatable = entity.isRotatable,
lastUser = entity.lastUser
)
}
}
@ -191,15 +199,20 @@ data class MapChunkResourceUpdate(
override val events: Map<EventName, LuaJsonList<Tick>>? = null,
val resourcesXY: FactorioEntityUpdateDictionary.Resource,
// Optimise JSON size by storing amounts separately.
val amounts: ResourceAmounts = ResourceAmounts(),
// Initial amount is only available for infinite resources, and infinite resources (oil) are more
// uncommon, so there are unlikely to be many per chunk
val initialAmounts: ResourceAmounts = ResourceAmounts(),
) : KafkatorioKeyedPacketData() {
fun resources(): List<FactorioEntityData.Resource> =
resourcesXY.map { (position, resource) ->
amounts.map { (position, amount) ->
FactorioEntityData.Resource(
protoId = key.protoId,
position = position,
element = resource,
amount = amount,
initialAmount = initialAmounts[position],
)
}
}

2
modules/events-library/src/jvmMain/kotlin/dev/adamko/kafkatorio/schema/kt2ts2.kt
View file

@ -99,7 +99,7 @@ class MySourceCodeGenerator(
(element.typeRef as? TsTypeRef.Literal)?.element is TsLiteral.Custom
|| element.id.name == "Tick"
|| element.id.name == "FactorioEntityUpdateEntityDictionary"
|| element.id.name == "FactorioEntityUpdateResourceDictionary"
|| element.id.name == "FactorioEntityUpdateResourceAmountDictionary"
) {
val aliasedRef = generateTypeReference(element.typeRef)
"""

3
modules/events-mod/src/generated/lua/main/emitting/PacketEmitter.lua
View file

@ -15,7 +15,8 @@ function KafkatorioPacketEmitter.prototype.emitKeyedPacket(self, data)
end
function KafkatorioPacketEmitter.emitPacket(self, packet)
local data = game.table_to_json(packet)
localised_print("KafkatorioPacket: " .. data)
local encoded = game.encode_string(data)
print("KafkatorioPacket encoded::: " .. tostring(encoded))
end
local PacketEmitter = __TS__New(____exports.KafkatorioPacketEmitter)
____exports.default = PacketEmitter

7
modules/events-mod/src/generated/lua/main/events/converters.lua
View file

@ -42,13 +42,6 @@ do
end
return masks
end
function Converters.convertResourceEntity(entity)
if entity.type ~= "resource" then
return nil
else
return {amount = entity.amount, initialAmount = entity.initial_amount}
end
end
local function convertEntityStatus(status)
if status == nil then
return nil

23
modules/events-mod/src/generated/lua/main/events/handlers/EntityUpdateHandler.lua
View file

@ -142,16 +142,23 @@ function EntityUpdatesHandler.throttleResourcesUpdate(self, eventName, eventTick
local ____data_events_eventName_17 = data.events[eventName]
____data_events_eventName_17[#____data_events_eventName_17 + 1] = eventTick
for ____, entity in ipairs(resourceEntities) do
local resourceUpdate = Converters.convertResourceEntity(entity)
if resourceUpdate ~= nil then
if data.resourcesXY == nil then
data.resourcesXY = {}
if data.amounts == nil then
data.amounts = {}
end
local ____data_resourcesXY_18, ____tostring_result_19 = data.resourcesXY, tostring(entity.position.x)
if ____data_resourcesXY_18[____tostring_result_19] == nil then
____data_resourcesXY_18[____tostring_result_19] = {}
local ____data_amounts_18, ____tostring_result_19 = data.amounts, tostring(entity.position.x)
if ____data_amounts_18[____tostring_result_19] == nil then
____data_amounts_18[____tostring_result_19] = {}
end
data.resourcesXY[tostring(entity.position.x)][tostring(entity.position.y)] = resourceUpdate
data.amounts[tostring(entity.position.x)][tostring(entity.position.y)] = entity.amount
if entity.initial_amount ~= nil then
if data.initialAmounts == nil then
data.initialAmounts = {}
end
local ____data_initialAmounts_20, ____tostring_result_21 = data.initialAmounts, tostring(entity.position.x)
if ____data_initialAmounts_20[____tostring_result_21] == nil then
____data_initialAmounts_20[____tostring_result_21] = {}
end
data.initialAmounts[tostring(entity.position.x)][tostring(entity.position.y)] = entity.initial_amount
end
end
end

28
modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts
View file

@ -40,8 +40,8 @@ export namespace KafkatorioPacketData {
type: KafkatorioPacketData.Type.ConfigurationUpdate;
factorioData: ConfigurationUpdateGameData;
allMods: ConfigurationUpdateModData[];
modStartupSettingsChange: boolean;
migrationApplied: boolean;
modStartupSettingsChange?: boolean;
migrationApplied?: boolean;
}
export interface ConsoleChatUpdate {
@ -94,7 +94,8 @@ export namespace KafkatorioPacketData {
type: KafkatorioPacketData.Type.MapChunkResourceUpdate;
key: MapChunkResourceUpdateKey;
events?: { [key: EventName]: Tick[] } | null;
resourcesXY: FactorioEntityUpdateResourceDictionary;
amounts?: FactorioEntityUpdateResourceAmountDictionary;
initialAmounts?: FactorioEntityUpdateResourceAmountDictionary;
}
export interface MapChunkTileUpdate {
@ -132,7 +133,7 @@ export namespace KafkatorioPacketData {
bannedReason?: string | null;
kickedReason?: string | null;
disconnectReason?: string | null;
isRemoved?: boolean | null;
isRemoved?: boolean;
}
export interface Error {
@ -171,7 +172,7 @@ export interface MapChunkEntityUpdateKey {
chunkPosition: MapChunkPosition;
}
export type FactorioEntityUpdateEntityDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateEntity } };
export type FactorioEntityUpdateEntityDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateElementStandard } };
export interface MapChunkResourceUpdateKey {
protoId: PrototypeId;
@ -179,7 +180,7 @@ export interface MapChunkResourceUpdateKey {
chunkPosition: MapChunkPosition;
}
export type FactorioEntityUpdateResourceDictionary = { [key: string]: { [key: string]: FactorioEntityUpdateResource } };
export type FactorioEntityUpdateResourceAmountDictionary = { [key: string]: { [key: string]: UInt } };
export interface MapChunkTileUpdateKey {
chunkPosition: MapChunkPosition;
@ -251,7 +252,7 @@ export namespace FactorioPrototype {
layer: UInt;
collisionMasks: string[];
order: string;
canBeMined: boolean;
canBeMined?: boolean;
}
}
@ -274,7 +275,7 @@ export interface EntityItemGroup {
}
export interface EntityMiningProperties {
canBeMined: boolean;
canBeMined?: boolean;
products: MinedProduct[] | null;
}
@ -293,21 +294,16 @@ export enum ChunkSize {
export type PrototypeKey = Int & { __PrototypeKey__: void };
export interface FactorioEntityUpdateEntity {
export interface FactorioEntityUpdateElementStandard {
status?: FactorioEntityStatus | null;
unitNumber?: UnitNumber | null;
graphicsVariation?: UByte | null;
health?: Float | null;
isActive?: boolean | null;
isRotatable?: boolean | null;
isActive?: boolean;
isRotatable?: boolean;
lastUser?: UInt | null;
}
export interface FactorioEntityUpdateResource {
amount: UInt;
initialAmount?: UInt | null;
}
export type MinedProduct =
| MinedProduct.MinedProductFluid
| MinedProduct.MinedProductItem;

6
modules/events-mod/src/main/typescript/main/emitting/PacketEmitter.ts
View file

@ -26,8 +26,10 @@ export class KafkatorioPacketEmitter {
/** Emit a serialised KafkatorioPacket */
private static emitPacket<T extends KafkatorioPacket>(packet: T) {
let data = game.table_to_json(packet)
localised_print(`KafkatorioPacket: ${data}`)
const data = game.table_to_json(packet)
// print(`KafkatorioPacket::: ${data}`)
const encoded = game.encode_string(data)
print(`KafkatorioPacket encoded::: ${encoded}`)
// rcon.print(`KafkatorioPacket: ${data}`)
}

15
modules/events-mod/src/main/typescript/main/events/converters.ts
View file

@ -5,7 +5,6 @@ import {
EntityMiningProperties,
EventName,
FactorioEntityStatus,
FactorioEntityUpdateResource,
KafkatorioPacketData,
MapBoundingBox,
MapChunkPosition,
@ -76,20 +75,6 @@ export namespace Converters {
}
export function convertResourceEntity(
entity: LuaEntity
): FactorioEntityUpdateResource | null {
if (entity.type != "resource") {
return null
} else {
return {
amount: entity.amount,
initialAmount: entity.initial_amount,
}
}
}
function convertEntityStatus(
status: defines.entity_status | undefined
): FactorioEntityStatus | null {

20
modules/events-mod/src/main/typescript/main/events/handlers/EntityUpdateHandler.ts
View file

@ -1,6 +1,6 @@
import {
EventName,
FactorioEntityUpdateEntity,
FactorioEntityUpdateElementStandard,
KafkatorioPacketData,
MapChunkPosition,
PrototypeId,
@ -94,7 +94,7 @@ export class EntityUpdatesHandler {
chunkPosition: Converters.mapPositionToChunkPosition(entity.position)
}
const entityUpdate: FactorioEntityUpdateEntity = {
const entityUpdate: FactorioEntityUpdateElementStandard = {
unitNumber: entity.unit_number,
graphicsVariation: entity.graphics_variation,
health: entity.health,
@ -140,7 +140,7 @@ export class EntityUpdatesHandler {
for (const [protoId, resourceEntities] of pairs(entitiesByProtoId)) {
// create an update for each prototype ID
// create a MapChunkResourceUpdate for each prototype ID
const entityUpdateKey: KafkatorioPacketData.MapChunkResourceUpdate["key"] = {
protoId: protoId,
@ -158,18 +158,20 @@ export class EntityUpdatesHandler {
data.events[eventName].push(eventTick)
for (const entity of resourceEntities) {
const resourceUpdate = Converters.convertResourceEntity(entity)
if (resourceUpdate != null) {
data.resourcesXY ??= {}
data.resourcesXY[`${entity.position.x}`] ??= {}
data.resourcesXY[`${entity.position.x}`][`${entity.position.y}`] = resourceUpdate
data.amounts ??= {}
data.amounts[`${entity.position.x}`] ??= {}
data.amounts[`${entity.position.x}`][`${entity.position.y}`] = entity.amount
if (entity.initial_amount != null) {
data.initialAmounts ??= {}
data.initialAmounts[`${entity.position.x}`] ??= {}
data.initialAmounts[`${entity.position.x}`][`${entity.position.y}`] = entity.initial_amount
}
}
}
)
}
}

9
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/KafkatorioTopology.kt
View file

@ -167,6 +167,7 @@ internal class KafkatorioTopology(
}
/** Receive log messages from the syslog socket, and send them to a Kafka topic */
private fun syslogProducer(
id: String = "syslog-producer"
) {
@ -187,10 +188,14 @@ internal class KafkatorioTopology(
}
syslogServer.messages
.filter { it.message?.startsWith("KafkatorioPacket:") == true }
.filter {
(it.message ?: "").run {
startsWith("KafkatorioPacket:::") || startsWith("KafkatorioPacket encoded:::")
}
}
.onEach { syslogMsg ->
val kafkatorioMessage = syslogMsg.message?.substringAfter("KafkatorioPacket:")
val kafkatorioMessage = syslogMsg.message?.substringAfter(":::")
val record = ProducerRecord<String, String>(
TOPIC_SRC_SERVER_LOG,

24
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/server/processor/topology/factorioServerPackets.kt
View file

@ -14,7 +14,13 @@ import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.streams.map
import dev.adamko.kotka.extensions.streams.to
import dev.adamko.kotka.kxs.serde
import java.util.zip.Inflater
import kotlinx.serialization.decodeFromString
import okio.Buffer
import okio.ByteString
import okio.ByteString.Companion.decodeBase64
import okio.buffer
import okio.inflate
import org.apache.kafka.common.serialization.Serdes
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.KStream
@ -33,7 +39,8 @@ fun factorioServerPacketStream(
)
).map("decode-packets") { serverId: String, value: String ->
val packet = runCatching {
jsonMapper.decodeFromString<KafkatorioPacket>(value)
val decoded = decodeFactorioEncodedString(value)
jsonMapper.decodeFromString<KafkatorioPacket>(decoded)
}.getOrElse { e ->
KafkatorioPacket(
modVersion = "unknown",
@ -69,3 +76,18 @@ fun splitFactorioServerPacketStream(
value.data.topicName
}
}
/**
* Factorio `game.encode_string` zlib encodes a string, then base64's it. Here we do the reverse.
*/
private fun decodeFactorioEncodedString(
source: String
): String {
val bytes: ByteString = source.decodeBase64() ?: error("failed to base64 decode $source")
return Buffer().write(bytes)
.inflate(Inflater())
.buffer()
.readString(Charsets.UTF_8)
}