lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

biggest refactor ever...

Browse source 
- use kxs-ts-gen for TypeScript schemas
- update schema files, so they're tidier (no need for typescript-generator workarounds)
- version bumps (kotlin, gradle, tstl, factorio, kotest)
- improve kxs binary format (use okio)
- move schema files package ('events' wasn't needed)
- tidy up topic creation, naming. base it off classes, make util functions
This commit is contained in:
Adam 2022-04-19 23:25:10 +02:00
parent db14865ca9
commit 3fff57d46d
82 changed files with 2545 additions and 1914 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -3,6 +3,7 @@
.secret.*
.idea
!.idea/icon.svg
node_modules

13
build.gradle.kts
View file

@ -8,7 +8,7 @@ plugins {
}
group = "dev.adamko.kafkatorio"
version = "0.3.2"
version = "0.4.0"
val licenseFile: RegularFile by extra(layout.projectDirectory.file("LICENSE"))
@ -35,7 +35,7 @@ idea {
}
tasks.wrapper {
gradleVersion = "7.4.1"
gradleVersion = "7.4.2"
distributionType = Wrapper.DistributionType.ALL
}
@ -57,3 +57,12 @@ val runKafkatorio by tasks.registering {
":modules:infra-factorio-server:processRestart",
)
}
val runEventsServer by tasks.registering {
group = project.name
dependsOn(
":modules:events-server:run",
)
}

4
buildSrc/build.gradle.kts
View file

@ -12,7 +12,7 @@ object Versions {
const val jvmTarget = "11"
const val kotlinTarget = "1.6"
const val kotlin = "1.6.10"
const val kotlin = "1.6.20"
const val gradleNodePlugin = "3.2.1"
const val gradleDockerComposePlugin = "0.15.1"
@ -22,7 +22,7 @@ object Versions {
const val semver = "1.1.2"
const val http4k = "4.19.1.0"
const val kotest = "5.1.0"
const val kotest = "5.2.3"
}

16
buildSrc/repositories.settings.gradle.kts
View file

@ -10,6 +10,7 @@ dependencyResolutionManagement {
// repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS)
repositories {
myMavenLocal()
mavenCentral()
jitpack()
gradlePluginPortal()
@ -31,6 +32,7 @@ dependencyResolutionManagement {
pluginManagement {
repositories {
myMavenLocal()
gradlePluginPortal()
mavenCentral()
jitpack()
@ -39,6 +41,20 @@ dependencyResolutionManagement {
}
fun RepositoryHandler.jitpack() {
maven("https://jitpack.io")
}
fun RepositoryHandler.myMavenLocal(enabled: Boolean = true) {
if (enabled) {
logger.lifecycle("Maven local is enabled")
mavenLocal {
content {
includeGroup("dev.adamko")
includeGroup("dev.adamko.kxtsgen")
}
}
}
}

6
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/common.kt
View file

@ -16,11 +16,11 @@ object Versions {
const val kotlinLang = "1.6"
const val jvmTarget = "11"
const val node = "14.19.0"
const val node = "14.19.1"
const val kotlinWrappers = "0.0.1-pre.300-kotlin-1.6.10"
const val kotlinWrappers = "0.0.1-pre.330-kotlin-1.6.20"
const val junit = "5.8.2"
const val kotest = "5.1.0"
const val kotest = "5.2.3"
const val mockk = "1.12.1"
}

1
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts
View file

@ -47,7 +47,6 @@ tasks.withType<KotlinCompile>().configureEach {
"-opt-in=kotlin.RequiresOptIn",
"-opt-in=kotlin.ExperimentalStdlibApi",
"-opt-in=kotlin.time.ExperimentalTime",
// "-Xextended-compiler-checks",
// "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
// "-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)

4
gradle.properties
View file

@ -1,5 +1,5 @@
kotlin.js.compiler=ir
kotlin.mpp.stability.nowarn=true
kotlin.mpp.enableGranularSourceSetsMetadata=true
kotlin.native.enableDependencyPropagation=false
#kotlin.mpp.enableGranularSourceSetsMetadata=true
#kotlin.native.enableDependencyPropagation=false
org.gradle.dependency.verification.console=verbose

17
gradle/libs.versions.toml
View file

@ -6,7 +6,7 @@
## compilation ##
jvm = "11"
kotlin = "1.6.10"
kotlin = "1.6.20"
## libs ##
camel-kafkaConnectors = "0.11.0"
@ -17,13 +17,14 @@ kvision = "5.8.2"
log4j2 = "2.17.2"
qosLogback = "1.2.11"
slf4j = "1.7.36"
okio = "3.1.0"
## test ##
junit = "5.8.2"
kotest = "5.1.0"
kotest = "5.2.3"
## infra ##
factorio = "1.1.53"
factorio = "1.1.57"
confluent-kafkaConnect = "7.0.1"
#########
@ -53,9 +54,9 @@ kotlinx-serialization-protobuf = { group = "org.jetbrains.kotlinx", name = "kotl
kotlinx-serialization-yaml = { group = "com.charleskorn.kaml", name = "kaml", version = "0.37.0" }
kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version = "1.6.0" }
kotlinx-coroutines = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version = "1.6.1" }
kotlinx-html = { group = "org.jetbrains.kotlinx", name = "kotlinx-html", version = "0.7.3" }
kotlinx-html = { group = "org.jetbrains.kotlinx", name = "kotlinx-html", version = "0.7.5" }
kotlinx-nodejs = { group = "org.jetbrains.kotlinx", name = "kotlinx-nodejs", version = "0.0.7" }
@ -103,6 +104,12 @@ skrapeit = { group = "it.skrape", name = "skrapeit", version = "1.2.0" }
scrimage-core = { group = "com.sksamuel.scrimage", name = "scrimage-core", version = "4.0.31" }
## okio ##
okio-bom = { group = "com.squareup.okio", name = "okio-bom", version.ref = "okio" }
okio-core = { group = "com.squareup.okio", name = "okio" }
#############
### test ###
#############

2
gradle/wrapper/gradle-wrapper.properties vendored
View file

@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-all.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

20
modules/events-mod/src/main/typescript/generated/kafkatorio-schema/config-update.d.ts vendored
View file

@ -1,20 +0,0 @@
// Generated by TypeScriptGenerator - do not edit this file manually
interface FactorioModInfo {
currentVersion: string | null;
modName: string;
previousVersion: string | null;
}
interface FactorioGameDataUpdate {
newVersion: string | null;
oldVersion: string | null;
}
interface FactorioConfigurationUpdate extends KafkatorioPacket {
allMods: FactorioModInfo[];
factorioData: FactorioGameDataUpdate;
modVersion: string;
packetType: PacketType;
tick: uint;
}

217
modules/events-mod/src/main/typescript/generated/kafkatorio-schema/kafkatorio-schema.ts Normal file
View file

@ -0,0 +1,217 @@
// Generated by TypeScriptGenerator - do not edit this file manually
export interface KafkatorioPacket2 {
modVersion: string;
tick: Tick;
data: KafkatorioPacketData2;
}
export type Tick = UInt;
export type KafkatorioPacketData2 =
| KafkatorioPacketData2.ConfigurationUpdate
| KafkatorioPacketData2.ConsoleChatUpdate
| KafkatorioPacketData2.ConsoleCommandUpdate
| KafkatorioPacketData2.EntityUpdate
| KafkatorioPacketData2.MapChunkUpdate
| KafkatorioPacketData2.PlayerUpdate
| KafkatorioPacketData2.PrototypesUpdate
| KafkatorioPacketData2.SurfaceUpdate;
export namespace KafkatorioPacketData2 {
export enum Type {
ConfigurationUpdate = "dev.adamko.kafkatorio.schema2.ConfigurationUpdate",
ConsoleChatUpdate = "dev.adamko.kafkatorio.schema2.ConsoleChatUpdate",
ConsoleCommandUpdate = "dev.adamko.kafkatorio.schema2.ConsoleCommandUpdate",
PrototypesUpdate = "dev.adamko.kafkatorio.schema2.PrototypesUpdate",
SurfaceUpdate = "dev.adamko.kafkatorio.schema2.SurfaceUpdate",
EntityUpdate = "dev.adamko.kafkatorio.schema2.EntityUpdate",
MapChunkUpdate = "dev.adamko.kafkatorio.schema2.MapChunkUpdate",
PlayerUpdate = "dev.adamko.kafkatorio.schema2.PlayerUpdate",
}
export interface ConfigurationUpdate {
type: KafkatorioPacketData2.Type.ConfigurationUpdate;
factorioData: ConfigurationUpdateGameData;
allMods: ConfigurationUpdateModData[];
modStartupSettingsChange: boolean;
migrationApplied: boolean;
}
export interface ConsoleChatUpdate {
type: KafkatorioPacketData2.Type.ConsoleChatUpdate;
authorPlayerIndex: PlayerIndex | null;
content: string;
}
export interface ConsoleCommandUpdate {
type: KafkatorioPacketData2.Type.ConsoleCommandUpdate;
authorPlayerIndex: PlayerIndex | null;
command: string;
parameters: string;
}
export interface PrototypesUpdate {
type: KafkatorioPacketData2.Type.PrototypesUpdate;
prototypes: FactorioPrototype2[];
}
export interface SurfaceUpdate {
type: KafkatorioPacketData2.Type.SurfaceUpdate;
index: SurfaceIndex;
daytime: Double;
name: string;
}
export interface EntityUpdate {
type: KafkatorioPacketData2.Type.EntityUpdate;
key: EntityUpdateKey;
eventCounts?: { [key: string]: UInt } | null;
chunkPosition?: MapChunkPosition | null;
graphicsVariation?: UByte | null;
health?: Float | null;
isActive?: boolean | null;
isRotatable?: boolean | null;
lastUser?: UInt | null;
localisedDescription?: string | null;
localisedName?: string | null;
prototype?: PrototypeName | null;
}
export interface MapChunkUpdate {
type: KafkatorioPacketData2.Type.MapChunkUpdate;
key: MapChunkUpdateKey;
eventCounts?: { [key: string]: UInt } | null;
player?: PlayerIndex | null;
robot?: EntityIdentifiersData | null;
force?: ForceIndex | null;
tileDictionary?: MapTileDictionary | null;
isDeleted?: boolean | null;
}
export interface PlayerUpdate {
type: KafkatorioPacketData2.Type.PlayerUpdate;
key: PlayerUpdateKey;
eventCounts?: { [key: string]: UInt } | null;
characterUnitNumber?: UnitNumber | null;
chatColour?: Colour | null;
colour?: Colour | null;
name?: string | null;
afkTime?: Tick | null;
ticksToRespawn?: Tick | null;
forceIndex?: ForceIndex | null;
isAdmin?: boolean | null;
isConnected?: boolean | null;
isShowOnMap?: boolean | null;
isSpectator?: boolean | null;
lastOnline?: Tick | null;
onlineTime?: Tick | null;
position?: MapEntityPosition | null;
surfaceIndex?: SurfaceIndex | null;
tag?: string | null;
diedCause?: EntityIdentifiersData | null;
bannedReason?: string | null;
kickedReason?: string | null;
disconnectReason?: string | null;
isRemoved?: boolean | null;
}
}
export type UInt = uint;
export interface ConfigurationUpdateGameData {
oldVersion?: string | null;
newVersion?: string | null;
}
export type PlayerIndex = UInt;
export type SurfaceIndex = UInt;
export type Double = double;
export interface EntityUpdateKey {
unitNumber: UnitNumber;
name: string;
protoType: string;
}
export type MapChunkPosition = [
x: Int,
y: Int,
];
export type UByte = uint8;
export type Float = float;
export type PrototypeName = string;
export interface MapChunkUpdateKey {
chunkPosition: MapChunkPosition;
surfaceIndex: SurfaceIndex;
}
export interface EntityIdentifiersData {
unitNumber?: UnitNumber | null;
name: string;
protoType: string;
}
export type ForceIndex = UInt;
export interface MapTileDictionary {
tilesXY: { [key: string]: { [key: string]: PrototypeKey } };
protos: { [key: PrototypeName]: PrototypeKey };
}
export interface PlayerUpdateKey {
index: PlayerIndex;
}
export type UnitNumber = UInt;
export interface Colour {
red?: Float;
green?: Float;
blue?: Float;
alpha?: Float;
}
export type MapEntityPosition = [
x: Double,
y: Double,
];
export type Int = int;
export interface ConfigurationUpdateModData {
modName: string;
currentVersion?: string | null;
previousVersion?: string | null;
}
export type FactorioPrototype2 =
| FactorioPrototype2.MapTile;
export namespace FactorioPrototype2 {
export enum Type {
MapTile = "dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2.MapTile",
}
export interface MapTile {
type: FactorioPrototype2.Type.MapTile;
name: PrototypeName;
layer: UInt;
mapColour: Colour;
collisionMasks: List;
order: string;
canBeMined: boolean;
}
}
export type Byte = int8;
export type List = any;
export type PrototypeKey = Int;

20
modules/events-mod/src/main/typescript/generated/kafkatorio-schema/object-data.d.ts vendored
View file

@ -1,20 +0,0 @@
// Generated by TypeScriptGenerator - do not edit this file manually
type ObjectName = "LuaSurface" | "ConsoleChatMessage";
interface FactorioObjectData {
objectName: ObjectName;
}
interface SurfaceData extends FactorioObjectData {
daytime: double;
index: uint;
name: string;
objectName: ObjectName;
}
interface ConsoleChatMessage extends FactorioObjectData {
authorPlayerIndex: uint | null;
content: string;
objectName: ObjectName;
}

24
modules/events-mod/src/main/typescript/generated/kafkatorio-schema/prototype.d.ts vendored
View file

@ -1,24 +0,0 @@
// Generated by TypeScriptGenerator - do not edit this file manually
type PrototypeObjectName = "LuaTilePrototype";
interface FactorioPrototype {
prototypeObjectName: PrototypeObjectName;
}
interface FactorioPrototypes extends KafkatorioPacket {
modVersion: string;
packetType: PacketType;
prototypes: FactorioPrototype[];
tick: uint;
}
interface MapTilePrototype extends FactorioPrototype {
canBeMined: boolean;
collisionMasks: string[];
layer: uint;
mapColour: Colour;
name: string;
order: string;
prototypeObjectName: PrototypeObjectName;
}

51
modules/events-mod/src/main/typescript/generated/kafkatorio-schema/schema.d.ts vendored
View file

@ -1,51 +0,0 @@
// Generated by TypeScriptGenerator - do not edit this file manually
type PacketType = "EVENT" | "CONFIG" | "PROTOTYPES" | "UPDATE";
interface KafkatorioPacket {
modVersion: string;
packetType: PacketType;
tick: uint;
}
interface FactorioEvent extends KafkatorioPacket {
data: FactorioObjectData;
eventType: string;
modVersion: string;
packetType: PacketType;
tick: uint;
}
interface Colour {
alpha: float;
blue: float;
green: float;
red: float;
}
interface EntityIdentifiers {
name: string;
type: string;
unitNumber: uint | null;
}
interface EntityIdentifiersData extends EntityIdentifiers {
name: string;
type: string;
unitNumber: uint | null;
}
interface MapEntityPosition {
x: double;
y: double;
}
interface MapChunkPosition {
x: int;
y: int;
}
interface MapTileDictionary {
protos: { [key: string]: string };
tilesXY: { [key: string]: { [key: string]: string } };
}

90
modules/events-mod/src/main/typescript/generated/kafkatorio-schema/update.d.ts vendored
View file

@ -1,90 +0,0 @@
// Generated by TypeScriptGenerator - do not edit this file manually
type FactorioEventUpdateType = "PLAYER" | "MAP_CHUNK" | "ENTITY";
interface FactorioEventUpdateKey {
updateType: FactorioEventUpdateType;
}
interface FactorioEventUpdate extends FactorioEventUpdateKey {
eventCounts: { [key: string]: uint } | null;
updateType: FactorioEventUpdateType;
}
interface FactorioEventUpdatePacket extends KafkatorioPacket {
modVersion: string;
packetType: PacketType;
tick: uint;
update: FactorioEventUpdate;
}
interface PlayerUpdateKey extends FactorioEventUpdateKey {
index: uint;
}
interface PlayerUpdate extends FactorioEventUpdate, PlayerUpdateKey {
afkTime: uint | null;
bannedReason: string | null;
characterUnitNumber: uint | null;
chatColour: Colour | null;
colour: Colour | null;
diedCause: EntityIdentifiersData | null;
disconnectReason: string | null;
eventCounts: { [key: string]: uint } | null;
forceIndex: uint | null;
index: uint;
isAdmin: boolean | null;
isConnected: boolean | null;
isRemoved: boolean | null;
isShowOnMap: boolean | null;
isSpectator: boolean | null;
kickedReason: string | null;
lastOnline: uint | null;
name: string | null;
onlineTime: uint | null;
position: MapEntityPosition | null;
surfaceIndex: uint | null;
tag: string | null;
ticksToRespawn: uint | null;
updateType: FactorioEventUpdateType;
}
interface EntityUpdateKey extends EntityIdentifiers, FactorioEventUpdateKey {
name: string;
type: string;
unitNumber: uint;
}
interface EntityUpdate extends FactorioEventUpdate, EntityUpdateKey {
chunkPosition: MapChunkPosition | null;
eventCounts: { [key: string]: uint } | null;
graphicsVariation: uint8 | null;
health: float | null;
isActive: boolean | null;
isRotatable: boolean | null;
lastUser: uint | null;
localisedDescription: string | null;
localisedName: string | null;
name: string;
prototype: string | null;
type: string;
unitNumber: uint;
updateType: FactorioEventUpdateType;
}
interface MapChunkUpdateKey extends FactorioEventUpdateKey {
chunkPosition: MapChunkPosition;
surfaceIndex: uint;
}
interface MapChunkUpdate extends FactorioEventUpdate, MapChunkUpdateKey {
chunkPosition: MapChunkPosition;
eventCounts: { [key: string]: uint } | null;
force: uint | null;
isDeleted: boolean | null;
player: uint | null;
robot: EntityIdentifiersData | null;
surfaceIndex: uint;
tileDictionary: MapTileDictionary | null;
updateType: FactorioEventUpdateType;
}

41
modules/events-mod/src/main/typescript/main/PacketEmitter.ts Normal file
View file

@ -0,0 +1,41 @@
import {
KafkatorioPacket2,
KafkatorioPacketData2
} from "../generated/kafkatorio-schema/kafkatorio-schema";
import {KafkatorioKeyedPacketData} from "./types";
export class KafkatorioPacketEmitter {
emitInstantPacket<DATA extends KafkatorioPacketData2>(
data: DATA,
) {
KafkatorioPacketEmitter.emitPacket<KafkatorioPacket2>({
data: data,
modVersion: global.MOD_VERSION,
tick: game.tick,
})
}
emitKeyedPacket<DATA extends KafkatorioKeyedPacketData>(
data: DATA,
) {
KafkatorioPacketEmitter.emitPacket<KafkatorioPacket2>({
data: data,
modVersion: global.MOD_VERSION,
tick: game.tick,
})
}
/** Emit a serialised KafkatorioPacket */
private static emitPacket<T extends KafkatorioPacket2>(packet: T) {
let data = game.table_to_json(packet)
localised_print(`KafkatorioPacket: ${data}`)
// rcon.print(`KafkatorioPacket: ${data}`)
}
}
const PacketEmitter = new KafkatorioPacketEmitter()
export default PacketEmitter

187
modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts vendored
View file

@ -1,10 +1,12 @@
import KafkatorioSettings from "../settings/KafkatorioSettings";
import {KafkatorioKeyedPacketData} from "../types";
export namespace EventUpdates {
declare const global: {
cache: LuaTable<string, CacheEntry<FactorioEventUpdateType>>,
/** Map the hash of the key, as a string, a cache entry */
cache: LuaTable<string, CacheEntry<KafkatorioKeyedPacketData>>,
}
@ -19,7 +21,7 @@ export namespace EventUpdates {
if (force == true || isAnythingUndefined) {
log("Initialising global.cache")
global.cache = new LuaTable<string, CacheEntry<FactorioEventUpdateType>>()
global.cache = new LuaTable<string, CacheEntry<KafkatorioKeyedPacketData>>()
}
log(`Finished initialising EventDataCache`)
}
@ -29,12 +31,13 @@ export namespace EventUpdates {
* Update the cache data without restarting the 'time-to-emit' countdown, so that the data won't
* be emitted more than once per {@link CacheEntry.expirationDurationTicks}.
*/
public throttle<TYPE extends FactorioEventUpdateType>(
key: CacheKey<TYPE>,
mutate: CacheDataMutator<TYPE>,
public throttle<PACKET extends KafkatorioKeyedPacketData>(
key: PacketKey<PACKET>,
type: PacketType<PACKET>,
mutate: CacheDataMutator<PACKET>,
expirationDurationTicks?: uint,
) {
this.update(key, mutate, false, expirationDurationTicks)
this.update(key, type, mutate, false, expirationDurationTicks)
}
@ -42,23 +45,26 @@ export namespace EventUpdates {
* Update the cache data and reset the 'time-to-emit' countdown, so the data won't be emitted
* until there's an inactivity gap of {@link CacheEntry.expirationDurationTicks}.
*/
public debounce<TYPE extends FactorioEventUpdateType>(
key: CacheKey<TYPE>,
mutate: CacheDataMutator<TYPE>,
public debounce<PACKET extends KafkatorioKeyedPacketData>(
key: PacketKey<PACKET>,
type: PacketType<PACKET>,
mutate: CacheDataMutator<PACKET>,
expirationDurationTicks?: uint,
) {
this.update(key, mutate, true, expirationDurationTicks)
this.update(key, type, mutate, true, expirationDurationTicks)
}
private update<TYPE extends FactorioEventUpdateType>(
key: CacheKey<TYPE>,
mutate: CacheDataMutator<TYPE>,
private update<PACKET extends KafkatorioKeyedPacketData>(
key: PacketKey<PACKET>,
type: PacketType<PACKET>,
mutate: CacheDataMutator<PACKET>,
resetLastUpdated: boolean,
expirationDurationTicks?: uint,
) {
const entry: CacheEntry<TYPE> = this.getCacheEntry(key) ?? this.createCacheEntry(key, mutate)
mutate(entry.data)
const entry: CacheEntry<PACKET> = this.getCacheEntry(key) ??
Manager.createCacheEntry(key, type)
mutate(entry.packet)
if (resetLastUpdated) {
entry.lastUpdatedTick = game.tick
}
@ -72,8 +78,8 @@ export namespace EventUpdates {
* Usage example: If an urgent update comes in, then the expiration can be reduced so that
* it's transmitted sooner.
*/
public setExpiration(
key: CacheKey<FactorioEventUpdateType>,
public setExpiration<PACKET extends KafkatorioKeyedPacketData>(
key: PacketKey<PACKET>,
expirationDurationTicks: uint,
) {
const entry = this.getCacheEntry(key)
@ -83,15 +89,16 @@ export namespace EventUpdates {
}
public extractExpired(): Array<FactorioEventUpdate> {
// use in/out v4.7.0 https://github.com/microsoft/TypeScript/pull/48240
public extractExpired(): Array<KafkatorioKeyedPacketData> {
let countExpired = 0
let countTotal = 0
const expiredData: Array<FactorioEventUpdate> = []
const expiredData: Array<KafkatorioKeyedPacketData> = []
for (let [key, entry] of global.cache) {
countTotal++
if (this.isExpired(entry)) {
countExpired++
expiredData.push(entry.data)
expiredData.push(entry.packet)
global.cache.delete(key)
}
}
@ -102,15 +109,15 @@ export namespace EventUpdates {
}
private getCacheEntry<TYPE extends FactorioEventUpdateType>(
key: CacheKey<TYPE>
): CacheEntry<TYPE> | undefined {
const hash = game.encode_string(game.table_to_json(key))!!
private getCacheEntry<PACKET extends KafkatorioKeyedPacketData>(
key: PacketKey<PACKET>
): CacheEntry<PACKET> | undefined {
const hash = Manager.hashKey(key)
if (!global.cache.has(hash)) {
return undefined
}
const value: CacheEntry<any> = global.cache.get(hash)
if (this.isEntryInstanceOf(value, key.updateType)) {
if (this.isEntryInstanceOf(value, key)) {
return value
} else {
// Type mismatch. This shouldn't happen...
@ -119,40 +126,53 @@ export namespace EventUpdates {
}
private createCacheEntry<TYPE extends FactorioEventUpdateType>(
key: CacheKey<TYPE>,
mutate: CacheDataMutator<TYPE>,
): CacheEntry<TYPE> {
const data: CacheData<TYPE> = <CacheData<TYPE>>{...key}
mutate(data)
let entry: CacheEntry<TYPE> = new CacheEntry<TYPE>(data)
const hash = game.encode_string(game.table_to_json(key))!!
private static createCacheEntry<PACKET extends KafkatorioKeyedPacketData>(
key: PacketKey<PACKET>,
type: PacketType<PACKET>,
): CacheEntry<PACKET> {
const data: PACKET = <PACKET>{
type: type,
key: key,
}
let entry: CacheEntry<PACKET> = new CacheEntry<PACKET>(data)
const hash = Manager.hashKey(key)
global.cache.set(hash, entry)
return entry
}
isExpired<TYPE extends FactorioEventUpdateType>(entry: CacheEntry<TYPE>): boolean {
const expiryDuration = entry.expirationDurationTicks
??
KafkatorioSettings.getEventCacheExpirationTicks(entry.data.updateType)
isExpired<TYPE extends KafkatorioKeyedPacketData>(entry: CacheEntry<TYPE>): boolean {
const expiryDuration =
entry.expirationDurationTicks
?? KafkatorioSettings.getEventCacheExpirationTicks(entry.packet)
if (expiryDuration == undefined) {
return true
} else {
return game.tick - entry.lastUpdatedTick > expiryDuration
}
isEntryInstanceOf<TYPE extends FactorioEventUpdateType>(
entry: CacheEntry<any> | undefined,
updateType: TYPE,
): entry is CacheEntry<TYPE> {
return this.isDataInstanceOf<TYPE>(entry?.data, updateType)
}
isDataInstanceOf<TYPE extends FactorioEventUpdateType>(
data: CacheData<any> | undefined,
updateType: TYPE,
): data is CacheData<TYPE> {
return data != undefined && data.updateType == updateType
isEntryInstanceOf<PACKET extends KafkatorioKeyedPacketData>(
entry: CacheEntry<any> | undefined,
key: PacketKey<PACKET>,
): entry is CacheEntry<PACKET> {
return this.isDataInstanceOf(entry?.packet, key)
}
isDataInstanceOf<PACKET extends KafkatorioKeyedPacketData>(
packet: PACKET | undefined,
key: PacketKey<PACKET>,
): packet is PACKET {
return packet != undefined && packet.key == key
}
private static hashKey<PACKET extends KafkatorioKeyedPacketData>(
key: PacketKey<PACKET>,
): string {
return game.encode_string(game.table_to_json(key))!!
}
@ -166,54 +186,56 @@ export namespace EventUpdates {
}
/** Map a {@link FactorioEventUpdateType} to a {@link FactorioEventUpdate} DTO */
type ConvertToUpdate<TYPE extends FactorioEventUpdateType> =
TYPE extends "PLAYER" ? PlayerUpdate :
TYPE extends "MAP_CHUNK" ? MapChunkUpdate :
TYPE extends "ENTITY" ? EntityUpdate :
never
// export type KeyedPacketType = Exclude<KafkatorioPacketDataType,
// "CONFIG" | "CONSOLE_CHAT" | "CONSOLE_COMMAND" | "PROTOTYPES" | "SURFACE">
// /** Map a {@link KeyedPacketType} to a {@link KafkatorioPacketData} DTO */
// type ConvertToUpdate<TYPE extends KafkatorioKeyedPacketData> =
// TYPE extends "PLAYER" ? PlayerUpdate :
// TYPE extends "MAP_CHUNK" ? MapChunkUpdate :
// TYPE extends "ENTITY" ? EntityUpdate :
// never
/** Map a {@link FactorioEventUpdateType} to a {@link FactorioEventUpdateKey} */
type ConvertToUpdateKey<TYPE extends FactorioEventUpdateType> =
TYPE extends "PLAYER" ? PlayerUpdateKey :
TYPE extends "MAP_CHUNK" ? MapChunkUpdateKey :
TYPE extends "ENTITY" ? EntityUpdateKey :
never
// /** Map a {@link KeyedPacketType} to a {@link KafkatorioKeyedPacketKey} */
// type ConvertToUpdateKey<TYPE extends KafkatorioKeyedPacketData> =
// TYPE extends "PLAYER" ? PlayerUpdateKey :
// TYPE extends "MAP_CHUNK" ? MapChunkUpdateKey :
// TYPE extends "ENTITY" ? EntityUpdateKey :
// never
/** Make `updateType` specific */
export type CacheKey<TYPE extends FactorioEventUpdateType> =
CacheTyped<TYPE>
& Omit<ConvertToUpdateKey<TYPE>, "updateType">
// /** Make `updateType` specific */
export type PacketKey<PACKET extends KafkatorioKeyedPacketData> = PACKET["key"]
export type PacketType<PACKET extends KafkatorioKeyedPacketData> = PACKET["type"]
//
//
// /** Make `updateType` specific */
// export type CacheData<TYPE extends KafkatorioKeyedPacketData> =
// CacheTyped<TYPE>
// & Omit<ConvertToUpdate<TYPE>, "updateType">
//
//
// export type CacheTyped<TYPE extends KafkatorioKeyedPacketData> = {
// readonly updateType: TYPE
// }
/** Make `updateType` specific */
export type CacheData<TYPE extends FactorioEventUpdateType> =
CacheTyped<TYPE>
& Omit<ConvertToUpdate<TYPE>, "updateType">
export type CacheDataMutator<PACKET extends KafkatorioKeyedPacketData> = (packet: PACKET) => void
export type CacheTyped<TYPE extends FactorioEventUpdateType> = {
readonly updateType: TYPE
}
export type CacheDataMutator<TYPE extends FactorioEventUpdateType> =
(data: CacheData<TYPE>) => void
class CacheEntry<TYPE extends FactorioEventUpdateType> {
class CacheEntry<PACKET extends KafkatorioKeyedPacketData> {
lastUpdatedTick: uint = game.tick
expirationDurationTicks?: uint
data: CacheData<TYPE>
packet: PACKET
constructor(
data: CacheData<TYPE>,
packet: PACKET,
expirationDurationTicks?: uint,
) {
this.data = data
this.packet = packet
this.expirationDurationTicks = expirationDurationTicks
}
}
@ -222,4 +244,5 @@ export namespace EventUpdates {
}
const EventUpdatesManager = new EventUpdates.Manager()
export default EventUpdatesManager

15
modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts vendored
View file

@ -1,23 +1,16 @@
import {emitPacket} from "../emitKafkatorioPacket";
import EventCacheService from "./EventDataCache";
import packetEmitter from "../PacketEmitter";
import {KafkatorioKeyedPacketData} from "../types";
/** Every 0.1 seconds */
script.on_nth_tick(6, (nthTick: NthTickEventData) => {
const cachedEvents: Array<FactorioEventUpdate> = EventCacheService.extractExpired()
const cachedEvents: Array<KafkatorioKeyedPacketData> = EventCacheService.extractExpired()
if (cachedEvents.length > 0) {
log(`nth tick ${nthTick.tick} has ${cachedEvents.length} events`)
for (const event of cachedEvents) {
const packet: FactorioEventUpdatePacket = {
tick: nthTick.tick,
modVersion: global.MOD_VERSION,
packetType: "UPDATE",
update: event,
}
emitPacket(packet)
packetEmitter.emitKeyedPacket(event)
}
}
})

28
modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts
View file

@ -1,25 +1,33 @@
import {emitPacket} from "../emitKafkatorioPacket";
import PacketEmitter from "../PacketEmitter";
import {
ConfigurationUpdateGameData,
ConfigurationUpdateModData,
KafkatorioPacketData2
} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import Type = KafkatorioPacketData2.Type;
import ConfigurationUpdate = KafkatorioPacketData2.ConfigurationUpdate;
export function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
emitPacket<FactorioConfigurationUpdate>({
modVersion: global.MOD_VERSION,
packetType: "CONFIG",
const configUpdateData: ConfigurationUpdate = {
type: Type.ConfigurationUpdate,
migrationApplied: changeData.migration_applied,
modStartupSettingsChange: changeData.mod_startup_settings_changed,
allMods: allMods(changeData),
factorioData: factorioData(changeData),
tick: game.tick,
})
factorioData: factorioData(changeData)
}
PacketEmitter.emitInstantPacket(configUpdateData)
}
function factorioData(changeData: ConfigurationChangedData): FactorioGameDataUpdate {
function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdateGameData {
return {
oldVersion: changeData.old_version ?? null,
newVersion: changeData.new_version ?? null,
}
}
function allMods(e: ConfigurationChangedData): FactorioModInfo[] {
function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] {
let namesOfChangedMods: string[] = Object.keys(e.mod_changes)
let namesOfCurrentMods: string[] = Object.keys(script.active_mods)
@ -33,7 +41,7 @@ function allMods(e: ConfigurationChangedData): FactorioModInfo[] {
let currentVer = script.active_mods[modName] ?? e.mod_changes[modName].new_version ?? null
let previousVer = e.mod_changes[modName]?.old_version ?? null
return <FactorioModInfo>{
return <ConfigurationUpdateModData>{
modName: modName,
currentVersion: currentVer,
previousVersion: previousVer,

31
modules/events-mod/src/main/typescript/main/config-update/prototypes.ts
View file

@ -1,30 +1,37 @@
import {Converters} from "../events/converters";
import {emitPacket} from "../emitKafkatorioPacket";
import PacketEmitter from "../PacketEmitter";
import {
FactorioPrototype2,
KafkatorioPacketData2
} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import Type = KafkatorioPacketData2.Type;
import PrototypesUpdate = KafkatorioPacketData2.PrototypesUpdate;
export function emitPrototypes() {
emitPacket<FactorioPrototypes>({
modVersion: global.MOD_VERSION,
packetType: "PROTOTYPES",
const data: PrototypesUpdate = {
prototypes: prototypes(),
tick: game.tick,
})
type: Type.PrototypesUpdate,
}
PacketEmitter.emitInstantPacket(data)
}
function prototypes(): FactorioPrototype[] {
let prototypes: FactorioPrototype[] = []
function prototypes(): FactorioPrototype2[] {
let prototypes: FactorioPrototype2[] = []
prototypes.push(...getMapTilePrototypes())
return prototypes
}
function getMapTilePrototypes(): MapTilePrototype[] {
let tiles: MapTilePrototype[] = []
function getMapTilePrototypes(): FactorioPrototype2.MapTile[] {
let tiles: FactorioPrototype2.MapTile[] = []
for (let [, tile] of game.tile_prototypes) {
tiles.push(
<MapTilePrototype>{
prototypeObjectName: tile.object_name,
<FactorioPrototype2.MapTile>{
type: FactorioPrototype2.Type.MapTile,
name: tile.name,
order: tile.order,

6
modules/events-mod/src/main/typescript/main/emitKafkatorioPacket.ts
View file

@ -1,6 +0,0 @@
/** Emit a serialised KafkatorioPacket */
export function emitPacket<T extends KafkatorioPacket>(packet: T) {
let data = game.table_to_json(packet)
localised_print(`KafkatorioPacket: ${data}`)
// rcon.print(`KafkatorioPacket: ${data}`)
}

18
modules/events-mod/src/main/typescript/main/events/converters.ts
View file

@ -1,4 +1,5 @@
import {EventName} from "../types";
import {Colour} from "../../generated/kafkatorio-schema/kafkatorio-schema";
export namespace Converters {
@ -62,15 +63,6 @@ export namespace Converters {
// return result
// }
export function surfaceToTable(surface: LuaSurface): SurfaceData {
return {
objectName: surface.object_name,
name: surface.name,
index: surface.index,
daytime: surface.daytime,
}
}
// export function mapEntityPosition(mapPos: MapPositionTable): MapEntityPosition {
// return {
@ -102,14 +94,6 @@ export namespace Converters {
}
}
export function consoleChat(content: string, playerIndex?: uint): ConsoleChatMessage {
return {
objectName: "ConsoleChatMessage",
authorPlayerIndex: playerIndex ?? null,
content: content
}
}
// export function convertTile(tile: LuaTile): MapTile {
// return {

12
modules/events-mod/src/main/typescript/main/events/emitEvent.ts
View file

@ -1,12 +0,0 @@
import {emitPacket} from "../emitKafkatorioPacket";
/** Emit a serialised event */
export function emitEvent<T extends FactorioObjectData>(eventData: T, tick: uint, eventType: string) {
emitPacket({
data: eventData,
packetType: "EVENT",
eventType: eventType,
modVersion: global.MOD_VERSION,
tick: tick,
})
}

28
modules/events-mod/src/main/typescript/main/events/handlers.ts
View file

@ -1,5 +1,6 @@
import {Converters} from "./converters"
import {emitEvent} from "./emitEvent";
import PacketEmitter from "../PacketEmitter";
import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import Type = KafkatorioPacketData2.Type;
// export function handlePlayerUpdate(tick: uint, eventType: string, playerIndex: uint) {
// let player: LuaPlayer = game.players[playerIndex]
@ -29,24 +30,23 @@ import {emitEvent} from "./emitEvent";
// emitEvent(table, tick, eventType)
// }
export function handleSurfaceUpdate(
tick: uint,
event: OnTickEvent,
eventType: string,
surface: LuaSurface,
) {
let table = Converters.surfaceToTable(surface)
emitEvent(table, tick, eventType)
const update: KafkatorioPacketData2.SurfaceUpdate = {
type: Type.SurfaceUpdate,
name: surface.name,
index: surface.index,
daytime: surface.daytime
}
PacketEmitter.emitInstantPacket(update)
}
export function handleConsoleChat(
tick: uint,
eventType: string,
playerIndex: uint | undefined,
message: string,
) {
let table = Converters.consoleChat(message, playerIndex)
emitEvent(table, tick, eventType)
}
// export function handleChunkUpdate(
// tick: uint,

31
modules/events-mod/src/main/typescript/main/events/onConsoleEventHandlers.ts Normal file
View file

@ -0,0 +1,31 @@
import PacketEmitter from "../PacketEmitter";
import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import Type = KafkatorioPacketData2.Type;
script.on_event(
defines.events.on_console_chat,
(event: OnConsoleChatEvent) => {
const update: KafkatorioPacketData2.ConsoleChatUpdate = {
type: Type.ConsoleChatUpdate,
authorPlayerIndex: event.player_index ?? null,
content: event.message
}
PacketEmitter.emitInstantPacket(update)
}
)
script.on_event(
defines.events.on_console_command,
(event: OnConsoleCommandEvent) => {
const update: KafkatorioPacketData2.ConsoleCommandUpdate = {
type: Type.ConsoleCommandUpdate,
authorPlayerIndex: event.player_index ?? null,
command: event.command,
parameters: event.parameters,
}
PacketEmitter.emitInstantPacket(update)
}
)

119
modules/events-mod/src/main/typescript/main/events/onMapChunkEventListeners.ts
View file

@ -1,20 +1,25 @@
import {isEventType} from "./eventTypeCheck";
import {Converters} from "./converters";
import EventUpdatesManager, {EventUpdates} from "../cache/EventDataCache";
import CacheKey = EventUpdates.CacheKey;
import CacheData = EventUpdates.CacheData;
import {
KafkatorioPacketData2,
MapChunkPosition,
MapTileDictionary
} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import CacheKey = EventUpdates.PacketKey;
import Type = KafkatorioPacketData2.Type;
type MapChunkUpdater = (data: CacheData<"MAP_CHUNK">) => void
type MapChunkUpdater = (data: KafkatorioPacketData2.MapChunkUpdate) => void
const mapProtoNameToKey: { [key: string]: string } = {}
const mapProtoNameToKey: { [key: string]: uint } = {}
let protoIndex: uint = 0
function getProtoKey(protoName: string): string {
function getProtoKey(protoName: string): uint {
if (mapProtoNameToKey[protoName] == undefined) {
mapProtoNameToKey[protoName] = `${protoIndex++}`
mapProtoNameToKey[protoName] = protoIndex++
}
return mapProtoNameToKey[protoName]
}
@ -22,7 +27,7 @@ function getProtoKey(protoName: string): string {
function mapTilesUpdateDebounce(
surface: LuaSurface | undefined,
chunkPosition: ChunkPositionTable,
chunkPosition: MapChunkPosition,
tiles: TileRead[],
eventName: string,
updater?: MapChunkUpdater,
@ -33,14 +38,14 @@ function mapTilesUpdateDebounce(
return
}
const key: CacheKey<"MAP_CHUNK"> = {
const key: KafkatorioPacketData2.MapChunkUpdate["key"] = {
surfaceIndex: surface.index,
chunkPosition: chunkPosition,
updateType: "MAP_CHUNK",
}
EventUpdatesManager.debounce<"MAP_CHUNK">(
EventUpdatesManager.debounce<KafkatorioPacketData2.MapChunkUpdate>(
key,
Type.MapChunkUpdate,
data => {
data.tileDictionary ??= <MapTileDictionary>{
tilesXY: {},
@ -53,7 +58,7 @@ function mapTilesUpdateDebounce(
const yString = `${tile.position.y}`
const protoKey = getProtoKey(tile.name)
data.tileDictionary.protos[protoKey] = tile.name
data.tileDictionary.protos[tile.name] = protoIndex
data.tileDictionary.tilesXY[xString] ??= {}
data.tileDictionary.tilesXY[xString][yString] = protoKey
@ -105,7 +110,7 @@ export function handleChunkGeneratedEvent(
let tiles = getTiles(e.surface, e.area)
mapTilesUpdateDebounce(
e.surface,
e.position,
[e.position.x, e.position.y],
tiles,
Converters.eventNameString(e.name),
undefined,
@ -114,22 +119,22 @@ export function handleChunkGeneratedEvent(
}
script.on_event(
defines.events.on_chunk_charted,
(e: OnChunkChartedEvent) => {
log(`on_chunk_charted ${e.tick}`)
// const surface = getSurface(e.surface_index)
// if (surface == undefined) {
// return
// }
//
// let tiles = getTiles(surface, e.area)
// mapTilesUpdateDebounce(surface, e.position, tiles, (data => {
// data.force = e.force.index
// }))
}
)
// script.on_event(
// defines.events.on_chunk_charted,
// (e: OnChunkChartedEvent) => {
// log(`on_chunk_charted ${e.tick}`)
//
// // const surface = getSurface(e.surface_index)
// // if (surface == undefined) {
// // return
// // }
// //
// // let tiles = getTiles(surface, e.area)
// // mapTilesUpdateDebounce(surface, e.position, tiles, (data => {
// // data.force = e.force.index
// // }))
// }
// )
script.on_event(
@ -155,14 +160,19 @@ function groupTiles(tiles: TileRead[]): Map<MapChunkPosition, TileRead[]> {
const mapChunkPositionToTiles = new Map<MapChunkPosition, TileRead[]>()
for (const tile of tiles) {
const chunkPosition = {
x: math.floor((tile.position.x / 32)),
y: math.floor((tile.position.y / 32)),
}
const chunkPosition: MapChunkPosition = [
math.floor((tile.position.x / 32)),
math.floor((tile.position.y / 32)),
]
if (!mapChunkPositionToTiles.has(chunkPosition)) {
mapChunkPositionToTiles.set(chunkPosition, [])
}
mapChunkPositionToTiles.get(chunkPosition)?.push(tile)
// https://github.com/TypeScriptToLua/TypeScriptToLua/issues/1256
// mapChunkPositionToTiles.get(chunkPosition)?.push(tile)
const tiles = mapChunkPositionToTiles.get(chunkPosition)
if (tiles != null) {
tiles.push(tile)
}
}
return mapChunkPositionToTiles
}
@ -183,7 +193,7 @@ function convertOldPosition(
}
function onBuildTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent) {
function handleBuiltTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent) {
const surface = getSurface(event.surface_index)
if (surface == undefined) {
@ -209,7 +219,7 @@ function onBuildTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent)
data.robot = {
unitNumber: event.robot?.unit_number ?? null,
name: event.robot.name,
type: event.robot.type,
protoType: event.robot.type,
}
return
}
@ -225,28 +235,25 @@ function onBuildTileEvent(event: OnPlayerBuiltTileEvent | OnRobotBuiltTileEvent)
script.on_event(defines.events.on_player_built_tile, (e: OnPlayerBuiltTileEvent) => {
log(`on_player_built_tile ${e.tick}`)
onBuildTileEvent(e)
handleBuiltTileEvent(e)
})
script.on_event(defines.events.on_robot_built_tile, (e: OnRobotBuiltTileEvent) => {
log(`on_robot_built_tile ${e.tick}`)
onBuildTileEvent(e)
handleBuiltTileEvent(e)
})
// script.on_event(
// defines.events.on_player_mined_tile,
// (e: OnPlayerMinedTileEvent) => {
// // need to group tiles by ChunkPosition...
//
// }
// )
// script.on_event(
// defines.events.on_robot_mined_tile,
// (e: OnRobotMinedTileEvent) => {
// // need to group tiles by ChunkPosition...
//
// function handleMinedTileEvent(event: OnPlayerMinedTileEvent | OnRobotMinedTileEvent) {
// }
// )
//
// script.on_event(defines.events.on_player_mined_tile, (e: OnPlayerMinedTileEvent) => {
// log(`on_player_mined_tile ${e.tick}`)
// handleMinedTileEvent(e)
// })
// script.on_event(defines.events.on_robot_mined_tile, (e: OnRobotMinedTileEvent) => {
// log(`on_robot_mined_tile ${e.tick}`)
// handleMinedTileEvent(e)
// })
script.on_event(
@ -260,17 +267,15 @@ script.on_event(
}
for (const position of e.positions) {
const key: CacheKey<"MAP_CHUNK"> = {
const key: CacheKey<KafkatorioPacketData2.MapChunkUpdate> = {
surfaceIndex: surface.index,
chunkPosition: position,
updateType: "MAP_CHUNK",
chunkPosition: [position.x, position.y],
}
EventUpdatesManager.debounce<"MAP_CHUNK">(
EventUpdatesManager.debounce<KafkatorioPacketData2.MapChunkUpdate>(
key,
data => {
data.isDeleted = true
},
Type.MapChunkUpdate,
data => data.isDeleted = true,
0, // deletion is important - emit ASAP
)
}

21
modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts
View file

@ -1,17 +1,20 @@
import {Converters} from "./converters";
import EventUpdatesManager, {EventUpdates} from "../cache/EventDataCache";
import CacheData = EventUpdates.CacheData;
import EventUpdatesManager from "../cache/EventDataCache";
import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import Type = KafkatorioPacketData2.Type;
type PlayerUpdater = (player: LuaPlayer, data: CacheData<"PLAYER">) => void
type PlayerUpdater = (player: LuaPlayer, data: KafkatorioPacketData2.PlayerUpdate) => void
function playerUpdateThrottle(
playerIndex: uint,
eventName: string,
mutate: PlayerUpdater,
) {
EventUpdatesManager.throttle<"PLAYER">(
{index: playerIndex, updateType: "PLAYER"},
EventUpdatesManager.throttle<KafkatorioPacketData2.PlayerUpdate>(
{index: playerIndex},
Type.PlayerUpdate,
data => {
const player = game.players[playerIndex]
if (player != undefined) {
@ -36,7 +39,7 @@ function playerUpdateThrottle(
// )
// }
function playerOnlineInfo(player: LuaPlayer, data: CacheData<"PLAYER">) {
function playerOnlineInfo(player: LuaPlayer, data: KafkatorioPacketData2.PlayerUpdate) {
data.lastOnline = player.last_online
data.onlineTime = player.online_time
data.afkTime = player.afk_time
@ -73,7 +76,7 @@ script.on_event(
e.player_index,
Converters.eventNameString(e.name),
(player, data) => {
data.position = player.position
data.position = [player.position.x, player.position.y]
}
)
}
@ -86,7 +89,7 @@ script.on_event(
e.player_index,
Converters.eventNameString(e.name),
(player, data) => {
data.position = player.position
data.position = [player.position.x, player.position.y]
data.surfaceIndex = player.surface.index
}
)
@ -106,7 +109,7 @@ script.on_event(
data.diedCause = {
unitNumber: e.cause.unit_number ?? null,
name: e.cause.name,
type: e.cause.type
protoType: e.cause.type
}
}
}

24
modules/events-mod/src/main/typescript/main/events/registerHandlers.ts
View file

@ -1,9 +1,9 @@
import {handleConsoleChat, handleSurfaceUpdate,} from "./handlers";
import {Queue} from "../queue/queue";
import {isEventType} from "./eventTypeCheck";
import {Converters} from "./converters";
import KafkatorioSettings from "../settings/KafkatorioSettings";
import {handleChunkGeneratedEvent} from "./onMapChunkEventListeners";
import {handleSurfaceUpdate} from "./handlers";
// script.on_event(
@ -34,13 +34,8 @@ script.on_event(
(e: OnTickEvent) => {
if (e.tick % 1000 == 0) {
for (const [, surface] of game.surfaces) {
handleSurfaceUpdate(e.tick, Converters.eventNameString(e.name), surface)
handleSurfaceUpdate(e, Converters.eventNameString(e.name), surface)
}
// let packets = new KafkatorioPacketQueue().dequeueValues(1)
// for (const packet of packets) {
// emitPacket(packet)
// }
}
if (e.tick % 7 == 0) {
@ -64,21 +59,6 @@ script.on_event(
}
)
script.on_event(
defines.events.on_console_chat,
(e: OnConsoleChatEvent) => {
handleConsoleChat(e.tick, Converters.eventNameString(e.name), e.player_index, e.message)
}
)
// script.on_event(
// defines.events.on_chunk_generated,
// (e: OnChunkGeneratedEvent) => {
// handleChunkUpdate(e.tick, mapEventIdToName.get(e.name), e.surface.index, e.position,
// e.area) } ) script.on_event( [ defines.events.on_player_built_tile,
// defines.events.on_robot_built_tile, ], (builtTilesEvent) => { handleTilesUpdate(
// builtTilesEvent.tick, mapEventIdToName.get(builtTilesEvent.name), builtTilesEvent.surface_index,
// builtTilesEvent.tile, builtTilesEvent.tiles, ) } )
script.on_event(
defines.events.on_runtime_mod_setting_changed,

60
modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts
View file

@ -1,52 +1,57 @@
import {Data, DoubleSettingDefinition} from "typed-factorio/settings/types"
import {KafkatorioKeyedPacketData, KafkatorioKeyedPacketTypes} from "../types";
import {KafkatorioPacketData2} from "../../generated/kafkatorio-schema/kafkatorio-schema";
export class KafkatorioSettingsConfig {
private static eventCacheExpirationDefaultSeconds: Record<FactorioEventUpdateType, uint> = {
"PLAYER": 0.25,
"MAP_CHUNK": 30,
"ENTITY": 5,
private static eventCacheExpirationDefaultSeconds: Record<KafkatorioKeyedPacketTypes, double> = {
"dev.adamko.kafkatorio.schema2.PlayerUpdate": 0.5,
"dev.adamko.kafkatorio.schema2.MapChunkUpdate": 30,
"dev.adamko.kafkatorio.schema2.EntityUpdate": 5,
}
private eventCacheExpirationTicks: Record<FactorioEventUpdateType, double> = {
"PLAYER": KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds["PLAYER"],
"MAP_CHUNK": KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds["MAP_CHUNK"],
"ENTITY": KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds["ENTITY"],
private eventCacheExpirationTicks: Record<KafkatorioKeyedPacketTypes, double> = {
"dev.adamko.kafkatorio.schema2.PlayerUpdate":
KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData2.Type.PlayerUpdate],
"dev.adamko.kafkatorio.schema2.MapChunkUpdate":
KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData2.Type.MapChunkUpdate],
"dev.adamko.kafkatorio.schema2.EntityUpdate":
KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData2.Type.EntityUpdate],
}
public getEventCacheExpirationTicks(type: FactorioEventUpdateType): uint {
return this.eventCacheExpirationTicks[type]
public getEventCacheExpirationTicks(packet: KafkatorioKeyedPacketData): uint {
return this.eventCacheExpirationTicks[packet.type]
}
private static types: KafkatorioKeyedPacketTypes[] = [
KafkatorioPacketData2.Type.PlayerUpdate,
KafkatorioPacketData2.Type.MapChunkUpdate,
KafkatorioPacketData2.Type.EntityUpdate,
]
loadSettings(): void {
const types: FactorioEventUpdateType[] = ["PLAYER", "MAP_CHUNK", "ENTITY"]
for (const type of types) {
public loadSettings(): void {
for (const type of KafkatorioSettingsConfig.types) {
const seconds: double = settings.global[KafkatorioSettingsConfig.cacheDurationSettingName(type)].value as double
this.eventCacheExpirationTicks[type] = seconds * 60 // convert ticks to seconds
}
}
initialiseSettings(data: Data): void {
data.extend(
[
this.createDefaultCacheDuration("PLAYER"),
this.createDefaultCacheDuration("MAP_CHUNK"),
this.createDefaultCacheDuration("ENTITY"),
]
public initialiseSettings(data: Data): void {
const settings = KafkatorioSettingsConfig.types.map(
type => KafkatorioSettingsConfig.createDefaultCacheDuration(type)
)
data.extend(settings)
}
private createDefaultCacheDuration(
type: FactorioEventUpdateType,
private static createDefaultCacheDuration(
type: KafkatorioKeyedPacketTypes,
defaultExpirationSeconds: uint = KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[type],
): DoubleSettingDefinition {
return <DoubleSettingDefinition>{
return {
type: "double-setting",
name: KafkatorioSettingsConfig.cacheDurationSettingName(type),
setting_type: "runtime-global",
@ -58,13 +63,12 @@ export class KafkatorioSettingsConfig {
}
private static cacheDurationSettingName(type: FactorioEventUpdateType): string {
return `kafkatorio_event-cache_default-expiration-duration-seconds_${type}`
private static cacheDurationSettingName(type: KafkatorioKeyedPacketTypes): string {
return `kafkatorio_event-cache_default-expiration-duration-seconds_${type.replaceAll(".", "-")}`
}
}
const KafkatorioSettings = new KafkatorioSettingsConfig()
export default KafkatorioSettings

9
modules/events-mod/src/main/typescript/main/types.d.ts vendored
View file

@ -1,3 +1,5 @@
import {KafkatorioPacketData2} from "../generated/kafkatorio-schema/kafkatorio-schema";
export type EventName = keyof typeof defines.events
@ -15,3 +17,10 @@ type ExcludeNullKeys<TYPE> = Pick<TYPE, NonNullKeys<TYPE>>
type NonNullableKeys<TYPE> = {
[KEY in keyof TYPE]: NonNullable<TYPE[KEY]>;
}
type KafkatorioKeyedPacketData = Extract<KafkatorioPacketData2, { key: object }>
type KafkatorioKeyedPacketTypes = KafkatorioKeyedPacketData["type"];
type KafkatorioKeyedPacketKeys = KafkatorioKeyedPacketData["key"];

32
modules/events-mod/src/main/typescript/package-lock.json generated
View file

@ -1,13 +1,13 @@
{
"name": "kafkatorio-events-mod",
"version": "0.3.2",
"version": "0.4.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"enhanced-resolve": {
"version": "5.9.2",
"resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.9.2.tgz",
"integrity": "sha512-GIm3fQfwLJ8YZx2smuHpBKkXC1yOk+OBEmKckVyL0i/ea8mqDEykK3ld5dgH1QYPNyT/lIllxV2LULnxCHaHkA==",
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.9.3.tgz",
"integrity": "sha512-Bq9VSor+kjvW3f9/MiiR4eE3XYgOl7/rS8lnSxbRbF3kS0B2r+Y9w5krBWxZgDxASVZbdYrn5wT4j/Wb0J9qow==",
"requires": {
"graceful-fs": "^4.2.4",
"tapable": "^2.2.0"
@ -19,9 +19,9 @@
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
},
"graceful-fs": {
"version": "4.2.9",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz",
"integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ=="
"version": "4.2.10",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
"integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA=="
},
"has": {
"version": "1.0.3",
@ -75,19 +75,19 @@
"integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ=="
},
"typed-factorio": {
"version": "0.19.1",
"resolved": "https://registry.npmjs.org/typed-factorio/-/typed-factorio-0.19.1.tgz",
"integrity": "sha512-/iLa916Cw33vC6wgmCb4cPLjmvUA2zVviTx/vy80iEvpOHjoNkuHkGSsmvss1aItwUKUlHaLi3k0fK5OPiw1iA=="
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/typed-factorio/-/typed-factorio-1.0.0.tgz",
"integrity": "sha512-8JYdjizJKMUhEpaZ7SoVLQ+toe0XTebK/Zh/Iwwvs29x8Ml758XACX5Bn71LJK22u3obgu7YN2yn2GxZbQ554g=="
},
"typescript": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.2.tgz",
"integrity": "sha512-HM/hFigTBHZhLXshn9sN37H085+hQGeJHJ/X7LpBWLID/fbc2acUMfU+lGD98X81sKP+pFa9f0DZmCwB9GnbAg=="
"version": "4.6.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.3.tgz",
"integrity": "sha512-yNIatDa5iaofVozS/uQJEl3JRWLKKGJKh6Yaiv0GLGSuhpFJe7P3SbHZ8/yjAHRQwKRoA6YZqlfjXWmVzoVSMw=="
},
"typescript-to-lua": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/typescript-to-lua/-/typescript-to-lua-1.3.4.tgz",
"integrity": "sha512-9NkUOSUd9tufvwCGiWJ6Z1edizYcECUfjdZJZfDRKTEXKIddbAdV3IFR6u5gxvYYRfXhfNV6IKvqHS2HxHFs6w==",
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/typescript-to-lua/-/typescript-to-lua-1.4.3.tgz",
"integrity": "sha512-oGJO1BqXKSxdfwavfOzthelxaQVAc7ykJ4Z/BF96nAWz35nmuLyv9FOSS132TgzLmV02859ndMuTYZtiVnN6jQ==",
"requires": {
"enhanced-resolve": "^5.8.2",
"resolve": "^1.15.1",

22
modules/events-mod/src/main/typescript/package.json
View file

@ -1,28 +1,24 @@
{
"name": "kafkatorio-events-mod",
"version": "0.3.2",
"version": "0.4.0",
"private": true,
"scripts": {
"build": "tstl",
"dev": "tstl --watch"
},
"engines": {
"node": ">=10.13.0"
},
"devDependencies": {
"node": "~14.19.1"
},
"devDependencies": {},
"dependencies": {
"lua-types": "^2.11.0",
"typescript-to-lua": "1.3.4",
"typed-factorio": "0.19.1",
"typescript": "4.6.2"
"typescript-to-lua": "1.4.3",
"typed-factorio": "1.0.0",
"typescript": "4.6.3"
},
"// @types/node": "^12.0.8",
"license": "UNLICENSED",
"peerDependencies": {
},
"optionalDependencies": {
},
"bundledDependencies": [
]
"peerDependencies": {},
"optionalDependencies": {},
"bundledDependencies": []
}

18
modules/events-schema/build.gradle.kts
View file

@ -1,5 +1,6 @@
import dev.adamko.kafkatorio.gradle.asProvider
import dev.adamko.kafkatorio.gradle.typescriptAttributes
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
@ -11,6 +12,7 @@ plugins {
kotlin {
js(IR) {
binaries.executable()
// browser{} causes this error:
@ -32,6 +34,11 @@ kotlin {
nodejs()
}
jvm {
val main by compilations.getting {
kotlinOptions {
jvmTarget = "11"
}
}
testRuns["test"].executionTask.configure {
useJUnitPlatform()
}
@ -56,6 +63,10 @@ kotlin {
implementation(project.dependencies.platform(libs.kotlinx.serialization.bom))
implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json)
api("dev.adamko.kxtsgen:kxs-ts-gen-core:dev-SNAPSHOT")
// implementation("com.github.adamko-dev:kotlinx-serialization-typescript-generator:dev-SNAPSHOT") {
// isChanging = true
// }
}
}
@ -76,7 +87,9 @@ kotlin {
dependencies {
implementation("com.github.aSemy:ts-generator:v1.2.1")
// implementation("com.github.ntrrgc:ts-generator:1.1.2")
implementation("org.jetbrains.kotlin:kotlin-reflect")
implementation(kotlin("reflect"))
// implementation("com.github.adamko-dev:kotlinx-serialization-typescript-generator:0.0.5")
}
}
@ -105,7 +118,8 @@ val generateTypescript by tasks.registering(JavaExec::class) {
main.runtimeDependencyFiles
)
mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2tsKt")
// mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2tsKt")
mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2ts2Kt")
val tempOutputDirectory = file("$temporaryDir")
args(tempOutputDirectory)

57
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/converters/positions.kt
View file

@ -1,57 +0,0 @@
package dev.adamko.kafkatorio.events.schema.converters
import dev.adamko.kafkatorio.events.schema.MAP_CHUNK_SIZE
import dev.adamko.kafkatorio.events.schema.MapChunkPosition
import dev.adamko.kafkatorio.events.schema.MapEntityPosition
import dev.adamko.kafkatorio.events.schema.MapTilePosition
import kotlin.math.floor
operator fun MapTilePosition.times(factor: Int) =
MapTilePosition(x * factor, y * factor)
operator fun MapTilePosition.div(divisor: Int) =
MapTilePosition(x / divisor, y / divisor)
operator fun MapTilePosition.plus(addend: Int) =
MapTilePosition(x + addend, y + addend)
fun MapTilePosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
MapChunkPosition(
floor(x.toDouble() / chunkSize.toDouble()).toInt(),
floor(y.toDouble() / chunkSize.toDouble()).toInt(),
)
fun MapEntityPosition.toMapTilePosition() =
MapChunkPosition(floor(x).toInt(), floor(y).toInt())
fun MapEntityPosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
MapChunkPosition(
floor(x / chunkSize.toDouble()).toInt(),
floor(y / chunkSize.toDouble()).toInt(),
)
operator fun MapChunkPosition.times(factor: Int) =
MapChunkPosition(x * factor, y * factor)
operator fun MapChunkPosition.plus(addend: Int) =
MapChunkPosition(x + addend, y + addend)
operator fun MapChunkPosition.minus(subtrahend: Int) =
MapChunkPosition(x - subtrahend, y - subtrahend)
fun MapChunkPosition.leftTopTile(chunkSize: Int = MAP_CHUNK_SIZE): MapTilePosition =
MapTilePosition(
x * chunkSize,
y * chunkSize,
)
//val MapChunkPosition.rightBottomTile: MapTilePosition
// get() = leftTopTile + (MAP_CHUNK_SIZE - 1)
//
//
//val MapChunkPosition.boundingBox: MapBoundingBox
// get() = MapBoundingBox(leftTopTile, rightBottomTile)

158
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioCommonData.kt
View file

@ -1,158 +0,0 @@
package dev.adamko.kafkatorio.events.schema
import kotlin.jvm.JvmInline
import kotlinx.serialization.Serializable
const val MAP_CHUNK_SIZE = 32
@Serializable
@JvmInline
value class Tick(val value: UInt)
/**
* Coordinates of a [EntityData] on a map.
*
* A [MapEntityPosition] can be translated to a [MapChunkPosition] by dividing the `x`/`y` values
* by [32][MAP_CHUNK_SIZE].
*/
@Serializable
data class MapEntityPosition(
val x: Double,
val y: Double,
)
/** Coordinates of a chunk in a [SurfaceData] where each integer `x`/`y` represents a different
* chunk.
*
* A [MapChunkPosition] can be translated to a [MapEntityPosition] by multiplying the `x`/`y`
* values by [32][MAP_CHUNK_SIZE].
*/
@Serializable
data class MapChunkPosition(
val x: Int,
val y: Int,
)
/**
* Coordinates of a tile in a chunk on a [SurfaceData] where each integer `x`/`y` represents a
* different [MapTile].
*
* It rounds any `x`/`y` down to whole numbers.
*/
@Serializable
data class MapTilePosition(
val x: Int,
val y: Int,
)
@Serializable
data class MapBoundingBox(
val topLeft: MapTilePosition,
val bottomRight: MapTilePosition,
)
/**
* Red, green, blue and alpha values, all in range `[0, 1]` or all in range `[0, 255]` if any
* value is > 1.
*
* All values here are optional. Colour channels default to `0`, the alpha channel defaults to `1`.
*/
@Serializable
data class Colour(
val red: Float = 0f,
val green: Float = 0f,
val blue: Float = 0f,
val alpha: Float = 1f,
)
/** Size-efficient version of [Colour] (`4*4` bytes vs `4*1` bytes) */
@Serializable
data class ColourHex(
val red: UByte = UByte.MIN_VALUE,
val green: UByte = UByte.MIN_VALUE,
val blue: UByte = UByte.MIN_VALUE,
val alpha: UByte = UByte.MAX_VALUE,
) {
companion object {
val TRANSPARENT = ColourHex(0u, 0u, 0u, 0u)
}
}
@Serializable
data class MapTiles(
val surfaceIndex: SurfaceIndex,
val tiles: List<MapTile>,
)
@Serializable
data class MapTile(
val x: Int,
val y: Int,
val proto: PrototypeName,
)
interface EntityIdentifiers {
val unitNumber: UnitNumber?
val name: String
val type: String
}
// instant of EntityIdentifiers, because kxs needs a concrete instance
@Serializable
data class EntityIdentifiersData(
override val unitNumber: UnitNumber? = null,
override val name: String,
/** The prototype-type of the entity. */
override val type: String,
) : EntityIdentifiers
/**
* Size-optimised 2D map of x/y tile positions and prototype-name.
*
* Each X/Y coordinate maps to an arbitrary [PrototypeKey] (which is only valid for any specific
* instance). [protos] is specific to each instance of [MapTileDictionary], can be used to convert from a
*/
@Serializable
data class MapTileDictionary(
/** Map an X,Y coordinate a prototype name */
val tilesXY: Map<String, Map<String, PrototypeKey>>,
val protos: Map<PrototypeKey, PrototypeName>,
) {
fun toMapTileList(): List<MapTile> = buildList {
tilesXY.forEach { (xString, row) ->
row.forEach { (yString, protoIndex) ->
val x = xString.toIntOrNull()
val y = yString.toIntOrNull()
val protoName = protos[protoIndex]
if (x != null && y != null && protoName != null) {
add(MapTile(x, y, protoName))
}
}
}
}
companion object {
fun MapTileDictionary?.isNullOrEmpty(): Boolean {
return this == null || (tilesXY.isEmpty() && tilesXY.all { it.value.isEmpty() })
}
}
}
@Serializable
@JvmInline
value class PrototypeKey(val index: String)

287
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioEventUpdates.kt
View file

@ -1,287 +0,0 @@
package dev.adamko.kafkatorio.events.schema
import dev.adamko.kafkatorio.events.schema.FactorioUpdateData.FactorioUpdateDataType
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.EncodeDefault
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonContentPolymorphicSerializer
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
@Serializable
sealed class KafkatorioPacket {
/** Schema versioning */
abstract val modVersion: String
abstract val tick: Tick
abstract val update: FactorioUpdateData
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
sealed interface FactorioUpdateDataKey {
val updateType: FactorioUpdateDataType
}
@Serializable(with = FactorioUpdateData.JsonSerializer::class)
sealed class FactorioUpdateData : FactorioUpdateDataKey {
/** Discriminator for [FactorioUpdateData] */
@EncodeDefault
abstract override val updateType: FactorioUpdateDataType
/** the initial Factorio event (`defines.events`) trigger */
abstract val eventCounts: Map<String, UInt>?
@Serializable
enum class FactorioUpdateDataType {
PLAYER,
MAP_CHUNK,
ENTITY,
CONSOLE_CHAT,
CONSOLE_COMMAND,
SURFACE,
CONFIG,
PROTOTYPES,
;
companion object {
val values: List<FactorioUpdateDataType> = values().toList()
}
}
object JsonSerializer : JsonContentPolymorphicSerializer<FactorioUpdateData>(
FactorioUpdateData::class
) {
private val key = FactorioUpdateData::updateType.name
override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out FactorioUpdateData> {
val type = element
.jsonObject[key]
?.jsonPrimitive
?.contentOrNull
?.let { json ->
FactorioUpdateDataType.values.firstOrNull { it.name == json }
}
requireNotNull(type) { "Unknown FactorioEventUpdate $key: $element" }
return when (type) {
FactorioUpdateDataType.PLAYER -> PlayerUpdate.serializer()
FactorioUpdateDataType.MAP_CHUNK -> MapChunkUpdate.serializer()
FactorioUpdateDataType.ENTITY -> EntityUpdate.serializer()
FactorioUpdateDataType.CONSOLE_CHAT -> ConsoleChatUpdate.serializer()
FactorioUpdateDataType.CONSOLE_COMMAND -> ConsoleCommandUpdate.serializer()
FactorioUpdateDataType.SURFACE -> SurfaceUpdate.serializer()
FactorioUpdateDataType.CONFIG -> ConfigurationUpdate.serializer()
FactorioUpdateDataType.PROTOTYPES -> PrototypesUpdate.serializer()
}
}
}
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
sealed interface PlayerUpdateKey : FactorioUpdateDataKey {
val index: PlayerIndex
}
@Serializable
data class PlayerUpdate(
override val index: PlayerIndex,
override val eventCounts: Map<String, UInt>? = null,
val characterUnitNumber: UnitNumber? = null,
val chatColour: Colour? = null,
val colour: Colour? = null,
val name: String? = null,
val afkTime: Tick? = null,
val ticksToRespawn: Tick? = null,
val forceIndex: ForceIndex? = null,
val isAdmin: Boolean? = null,
val isConnected: Boolean? = null,
val isShowOnMap: Boolean? = null,
val isSpectator: Boolean? = null,
val lastOnline: Tick? = null,
val onlineTime: Tick? = null,
val position: MapEntityPosition? = null,
val surfaceIndex: SurfaceIndex? = null,
val tag: String? = null,
val diedCause: EntityIdentifiersData? = null,
val bannedReason: String? = null,
val kickedReason: String? = null,
val disconnectReason: String? = null,
/** `true` when a player is removed (deleted) from the game */
val isRemoved: Boolean? = null,
) : FactorioUpdateData(), PlayerUpdateKey {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.PLAYER
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
sealed interface EntityUpdateKey : EntityIdentifiers, FactorioUpdateDataKey {
/** A [unitNumber] is required for caching */
override val unitNumber: UnitNumber
override val name: String
override val type: String
}
@Serializable
data class EntityUpdate(
override val unitNumber: UnitNumber,
override val name: String,
override val type: String,
override val eventCounts: Map<String, UInt>? = null,
val chunkPosition: MapChunkPosition? = null,
val graphicsVariation: UByte? = null,
val health: Float? = null,
val isActive: Boolean? = null,
val isRotatable: Boolean? = null,
val lastUser: UInt? = null,
val localisedDescription: String? = null,
val localisedName: String? = null,
val prototype: PrototypeName? = null,
) : FactorioUpdateData(), EntityUpdateKey {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.ENTITY
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
sealed interface MapChunkUpdateKey : FactorioUpdateDataKey {
val chunkPosition: MapChunkPosition
val surfaceIndex: SurfaceIndex
}
@Serializable
data class MapChunkUpdate(
override val chunkPosition: MapChunkPosition,
override val surfaceIndex: SurfaceIndex,
override val eventCounts: Map<String, UInt>? = null,
val player: PlayerIndex? = null,
val robot: EntityIdentifiersData? = null,
val force: ForceIndex? = null,
/** updated tiles - might be partial and not all tiles in the chunk */
val tileDictionary: MapTileDictionary? = null,
val isDeleted: Boolean? = null,
) : FactorioUpdateData(), MapChunkUpdateKey {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.MAP_CHUNK
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@Serializable
data class ConsoleChatUpdate(
override val eventCounts: Map<String, UInt>? = null,
val authorPlayerIndex: PlayerIndex?,
val content: String,
) : FactorioUpdateData() {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.CONSOLE_CHAT
}
@Serializable
data class ConsoleCommandUpdate(
override val eventCounts: Map<String, UInt>? = null,
val authorPlayerIndex: PlayerIndex?,
val command: String,
val parameters: String,
) : FactorioUpdateData() {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.CONSOLE_COMMAND
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
sealed interface SurfaceUpdateKey : FactorioUpdateDataKey {
val index: SurfaceIndex
}
@Serializable
data class SurfaceUpdate(
override val index: SurfaceIndex,
override val eventCounts: Map<String, UInt>? = null,
val daytime: Double,
val name: String,
) : FactorioUpdateData(), SurfaceUpdateKey {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.SURFACE
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@Serializable
data class ConfigurationUpdate(
override val eventCounts: Map<String, UInt>? = null,
val factorioData: GameData,
val allMods: List<ModData>,
) : FactorioUpdateData() {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.CONFIG
@Serializable
data class GameData(
val oldVersion: String? = null,
val newVersion: String? = null,
)
@Serializable
data class ModData(
val modName: String,
val currentVersion: String? = null,
val previousVersion: String? = null,
)
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
@Serializable
data class PrototypesUpdate(
override val eventCounts: Map<String, UInt>? = null,
val prototypes: List<FactorioPrototype>,
) : FactorioUpdateData() {
@EncodeDefault
override val updateType: FactorioUpdateDataType = FactorioUpdateDataType.PROTOTYPES
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

38
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/converters/colour.kt → modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/colours.kt
View file

@ -1,8 +1,38 @@
package dev.adamko.kafkatorio.events.schema.converters
package dev.adamko.kafkatorio.schema.common
import dev.adamko.kafkatorio.events.schema.Colour
import dev.adamko.kafkatorio.events.schema.ColourHex
import kotlin.math.roundToInt
import kotlinx.serialization.Serializable
/**
* Red, green, blue and alpha values, all in range `[0, 1]` or all in range `[0, 255]` if any
* value is > 1.
*
* All values here are optional. Colour channels default to `0`, the alpha channel defaults to `1`.
*/
@Serializable
data class Colour(
val red: Float = 0f,
val green: Float = 0f,
val blue: Float = 0f,
val alpha: Float = 1f,
)
/** Size-efficient version of [Colour] (`4*4` bytes vs `4*1` bytes) */
@Serializable
data class ColourHex(
val red: UByte = UByte.MIN_VALUE,
val green: UByte = UByte.MIN_VALUE,
val blue: UByte = UByte.MIN_VALUE,
val alpha: UByte = UByte.MAX_VALUE,
) {
companion object {
val TRANSPARENT = ColourHex(0u, 0u, 0u, 0u)
}
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/** True if any value is greater than 1, so the values are hexadecimal. */
@ -74,8 +104,10 @@ fun Colour.toHex(): ColourHex {
)
}
private fun UByte.toPercentileFloat() = (this.toFloat() / ubyteMaxFloat).coerceIn(0f..1f)
fun ColourHex.toPercentile(): Colour {
return Colour(
red.toPercentileFloat(),

24
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioIds.kt → modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/factorioIds.kt
View file

@ -1,9 +1,14 @@
package dev.adamko.kafkatorio.events.schema
package dev.adamko.kafkatorio.schema.common
import kotlin.jvm.JvmInline
import kotlinx.serialization.Serializable
@Serializable
@JvmInline
value class Tick(val value: UInt)
/**
* Unique ID associated with this surface
*
@ -44,3 +49,20 @@ value class UnitNumber(private val id: UInt) {
value class PrototypeName(private val id: String) {
override fun toString(): String = id
}
interface EntityIdentifiers {
val unitNumber: UnitNumber?
val name: String
/** The prototype-type of the entity. */
val protoType: String
}
// instant of EntityIdentifiers, because kxs needs a concrete instance
@Serializable
data class EntityIdentifiersData(
override val unitNumber: UnitNumber? = null,
override val name: String,
override val protoType: String,
) : EntityIdentifiers

75
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/mapTile.kt Normal file
View file

@ -0,0 +1,75 @@
package dev.adamko.kafkatorio.schema.common
import dev.adamko.kafkatorio.schema.common.MapTileDictionary.PrototypeKey
import kotlin.jvm.JvmInline
import kotlinx.serialization.Serializable
const val MAP_CHUNK_SIZE = 32
@Serializable
data class MapTiles(
val surfaceIndex: SurfaceIndex,
val tiles: List<MapTile>,
)
@Serializable
data class MapTile(
val x: Int,
val y: Int,
val proto: PrototypeName,
)
/**
* 2D map of x/y tile positions and prototype-name, optimised for JSON size.
*
* [protos] is a one-to-one map between a [PrototypeName] and [PrototypeKey].
*
* [PrototypeKey] is an arbitrary integer value, which maps to one [PrototypeName] only for a
* specific instance of [MapTileDictionary].
*
* Each X/Y coordinate in [tilesXY] maps to a [PrototypeKey].
*/
@Serializable
data class MapTileDictionary(
/** Map an X,Y coordinate a prototype name */
val tilesXY: Map<String, Map<String, PrototypeKey>>,
val protos: Map<PrototypeName, PrototypeKey>,
) {
private val protosIndexToName: Map<PrototypeKey, PrototypeName> by lazy {
protos.entries.associate { (name, key) -> key to name }
}
fun toMapTileList(): List<MapTile> = buildList {
tilesXY.forEach { (xString, row) ->
row.forEach { (yString, protoIndex) ->
val x = xString.toIntOrNull()
val y = yString.toIntOrNull()
val protoName = protosIndexToName[protoIndex]
if (x != null && y != null && protoName != null) {
add(MapTile(x, y, protoName))
}
}
}
}
/** The collection-index of a prototype in [MapTileDictionary.protos] */
@Serializable
@JvmInline
value class PrototypeKey(val index: Int)
companion object {
fun MapTileDictionary?.isNullOrEmpty(): Boolean {
return this == null || (tilesXY.isEmpty() && tilesXY.all { it.value.isEmpty() })
}
}
}
@Serializable
data class MapBoundingBox(
val topLeft: MapTilePosition,
val bottomRight: MapTilePosition,
)

142
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/positions.kt Normal file
View file

@ -0,0 +1,142 @@
package dev.adamko.kafkatorio.schema.common
import dev.adamko.kxstsgen.core.*
import dev.adamko.kxstsgen.core.experiments.TupleSerializer
import kotlin.math.floor
import kotlinx.serialization.Serializable
/**
* Coordinates of a [EntityData] on a map.
*
* A [MapEntityPosition] can be translated to a [MapChunkPosition] by dividing the `x`/`y` values
* by [32][MAP_CHUNK_SIZE].
*/
@Serializable(with = MapEntityPosition.Serializer::class)
data class MapEntityPosition(
val x: Double,
val y: Double,
) {
object Serializer : TupleSerializer<MapEntityPosition>(
"MapEntityPosition",
{
element(MapEntityPosition::x)
element(MapEntityPosition::y)
}
) {
override fun tupleConstructor(elements: Iterator<*>): MapEntityPosition {
return MapEntityPosition(
elements.next() as Double,
elements.next() as Double,
)
}
}
}
/** Coordinates of a chunk in a [SurfaceData] where each integer `x`/`y` represents a different
* chunk.
*
* A [MapChunkPosition] can be translated to a [MapEntityPosition] by multiplying the `x`/`y`
* values by [32][MAP_CHUNK_SIZE].
*/
@Serializable(with = MapChunkPosition.Serializer::class)
data class MapChunkPosition(
val x: Int,
val y: Int,
) {
object Serializer : TupleSerializer<MapChunkPosition>(
"MapChunkPosition",
{
element(MapChunkPosition::x)
element(MapChunkPosition::y)
}
) {
override fun tupleConstructor(elements: Iterator<*>): MapChunkPosition {
return MapChunkPosition(
elements.next() as Int,
elements.next() as Int,
)
}
}
}
/**
* Coordinates of a tile in a chunk on a [SurfaceData] where each integer `x`/`y` represents a
* different [MapTile].
*
* It rounds any `x`/`y` down to whole numbers.
*/
@Serializable(with = MapTilePosition.Serializer::class)
data class MapTilePosition(
val x: Int,
val y: Int,
) {
object Serializer : TupleSerializer<MapTilePosition>(
"MapTilePosition",
{
element(MapTilePosition::x)
element(MapTilePosition::y)
}
) {
override fun tupleConstructor(elements: Iterator<*>): MapTilePosition {
val x = requireNotNull(elements.next() as? Int)
val y = requireNotNull(elements.next() as? Int)
return MapTilePosition(x, y)
}
}
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
operator fun MapTilePosition.times(factor: Int) =
MapTilePosition(x * factor, y * factor)
operator fun MapTilePosition.div(divisor: Int) =
MapTilePosition(x / divisor, y / divisor)
operator fun MapTilePosition.plus(addend: Int) =
MapTilePosition(x + addend, y + addend)
fun MapTilePosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
MapChunkPosition(
floor(x.toDouble() / chunkSize.toDouble()).toInt(),
floor(y.toDouble() / chunkSize.toDouble()).toInt(),
)
fun MapEntityPosition.toMapTilePosition() =
MapChunkPosition(floor(x).toInt(), floor(y).toInt())
fun MapEntityPosition.toMapChunkPosition(chunkSize: Int = MAP_CHUNK_SIZE) =
MapChunkPosition(
floor(x / chunkSize.toDouble()).toInt(),
floor(y / chunkSize.toDouble()).toInt(),
)
operator fun MapChunkPosition.times(factor: Int) =
MapChunkPosition(x * factor, y * factor)
operator fun MapChunkPosition.plus(addend: Int) =
MapChunkPosition(x + addend, y + addend)
operator fun MapChunkPosition.minus(subtrahend: Int) =
MapChunkPosition(x - subtrahend, y - subtrahend)
fun MapChunkPosition.leftTopTile(chunkSize: Int = MAP_CHUNK_SIZE): MapTilePosition =
MapTilePosition(
x * chunkSize,
y * chunkSize,
)
//val MapChunkPosition.rightBottomTile: MapTilePosition
// get() = leftTopTile + (MAP_CHUNK_SIZE - 1)
//
//
//val MapChunkPosition.boundingBox: MapBoundingBox
// get() = MapBoundingBox(leftTopTile, rightBottomTile)

68
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/events/kafkatorioPacket.kt Normal file
View file

@ -0,0 +1,68 @@
package dev.adamko.kafkatorio.schema.events
import dev.adamko.kafkatorio.schema.SerializerProvider
import dev.adamko.kafkatorio.schema.SerializerProviderDeserializer
import dev.adamko.kafkatorio.schema.common.Tick
import kotlin.reflect.KClass
import kotlin.reflect.KType
import kotlin.reflect.typeOf
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
//@Serializable
//data class KafkatorioPacket(
// val modVersion: String,
// val tick: Tick,
// val data: KafkatorioPacketData,
//)
@Serializable(with = KafkatorioPacketData.JsonSerializer::class)
sealed class KafkatorioPacketData {
abstract val updateType: KafkatorioPacketDataType
object JsonSerializer : SerializerProviderDeserializer<KafkatorioPacketData>(
KafkatorioPacketData::class,
KafkatorioPacketData::updateType.name,
{ json -> KafkatorioPacketDataType.entries.firstOrNull { it.name == json } }
)
}
/** Discriminator for [KafkatorioPacketData] */
@Serializable
enum class KafkatorioPacketDataType(
override val serializer: KSerializer<out KafkatorioPacketData>
) : SerializerProvider<KafkatorioPacketData> {
// non-keyed
CONFIG(ConfigurationUpdate.serializer()),
CONSOLE_CHAT(ConsoleChatUpdate.serializer()),
CONSOLE_COMMAND(ConsoleCommandUpdate.serializer()),
PROTOTYPES(PrototypesUpdate.serializer()),
SURFACE(SurfaceUpdate.serializer()),
// keyed
ENTITY(EntityUpdate.serializer()),
MAP_CHUNK(MapChunkUpdate.serializer()),
PLAYER(PlayerUpdate.serializer()),
;
companion object {
val entries: Set<KafkatorioPacketDataType> = values().toSet()
}
}
//inline fun <reified T: KafkatorioPacketData> KafkatorioPacketDataType.isType (): KClass<out T> {
// return when (this) {
// KafkatorioPacketDataType.CONFIG -> ConfigurationUpdate::class
// KafkatorioPacketDataType.CONSOLE_CHAT -> ConsoleChatUpdate::class
// KafkatorioPacketDataType.CONSOLE_COMMAND -> ConsoleCommandUpdate::class
// KafkatorioPacketDataType.PROTOTYPES -> PrototypesUpdate::class
// KafkatorioPacketDataType.SURFACE -> SurfaceUpdate::class
// KafkatorioPacketDataType.ENTITY -> EntityUpdate::class
// KafkatorioPacketDataType.MAP_CHUNK -> MapChunkUpdate::class
// KafkatorioPacketDataType.PLAYER -> PlayerUpdate::class
// }
//}

95
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/events/kafkatorioPacketInstant.kt Normal file
View file

@ -0,0 +1,95 @@
package dev.adamko.kafkatorio.schema.events
import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
import kotlinx.serialization.EncodeDefault
import kotlinx.serialization.Serializable
/** No key, no caching. Must be sent instantly. */
sealed class KafkatorioInstantPacketData : KafkatorioPacketData()
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class ConfigurationUpdate(
val factorioData: ConfigurationUpdateGameData,
val allMods: List<ConfigurationUpdateModData>,
val modStartupSettingsChange: Boolean,
val migrationApplied: Boolean,
) : KafkatorioInstantPacketData() {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.CONFIG
@Serializable
data class ConfigurationUpdateGameData(
val oldVersion: String? = null,
val newVersion: String? = null,
)
@Serializable
data class ConfigurationUpdateModData(
val modName: String,
val currentVersion: String? = null,
val previousVersion: String? = null,
)
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class ConsoleChatUpdate(
val authorPlayerIndex: PlayerIndex?,
val content: String,
) : KafkatorioInstantPacketData() {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.CONSOLE_CHAT
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class ConsoleCommandUpdate(
val authorPlayerIndex: PlayerIndex?,
val command: String,
val parameters: String,
) : KafkatorioInstantPacketData() {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.CONSOLE_COMMAND
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class PrototypesUpdate(
val prototypes: List<FactorioPrototype>,
) : KafkatorioInstantPacketData() {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.PROTOTYPES
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class SurfaceUpdate(
val index: SurfaceIndex,
val daytime: Double,
val name: String,
) : KafkatorioInstantPacketData() {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.SURFACE
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */

135
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/events/kafkatorioPacketKeyed.kt Normal file
View file

@ -0,0 +1,135 @@
package dev.adamko.kafkatorio.schema.events
import dev.adamko.kafkatorio.schema.common.Colour
import dev.adamko.kafkatorio.schema.common.EntityIdentifiers
import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
import dev.adamko.kafkatorio.schema.common.ForceIndex
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapEntityPosition
import dev.adamko.kafkatorio.schema.common.MapTileDictionary
import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.PrototypeName
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.common.UnitNumber
import kotlinx.serialization.EncodeDefault
import kotlinx.serialization.Serializable
sealed interface KafkatorioKeyedPacketKey
/** Has a key, so it can be debounced */
sealed class KafkatorioKeyedPacketData : KafkatorioPacketData() {
/** Count how many events this packet is aggregating data from */
abstract val eventCounts: Map<String, UInt>?
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
sealed interface EntityUpdateKey : EntityIdentifiers, KafkatorioKeyedPacketKey {
/** A [unitNumber] is required for caching */
override val unitNumber: UnitNumber
override val name: String
override val protoType: String
}
@Serializable
data class EntityUpdate(
override val unitNumber: UnitNumber,
override val name: String,
override val protoType: String,
override val eventCounts: Map<String, UInt>? = null,
val chunkPosition: MapChunkPosition? = null,
val graphicsVariation: UByte? = null,
val health: Float? = null,
val isActive: Boolean? = null,
val isRotatable: Boolean? = null,
val lastUser: UInt? = null,
val localisedDescription: String? = null,
val localisedName: String? = null,
val prototype: PrototypeName? = null,
) : KafkatorioKeyedPacketData(), EntityUpdateKey {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.ENTITY
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
sealed interface MapChunkUpdateKey : KafkatorioKeyedPacketKey {
val chunkPosition: MapChunkPosition
val surfaceIndex: SurfaceIndex
}
@Serializable
data class MapChunkUpdate(
override val chunkPosition: MapChunkPosition,
override val surfaceIndex: SurfaceIndex,
override val eventCounts: Map<String, UInt>? = null,
val player: PlayerIndex? = null,
val robot: EntityIdentifiersData? = null,
val force: ForceIndex? = null,
/** updated tiles - might be partial and not all tiles in the chunk */
val tileDictionary: MapTileDictionary? = null,
val isDeleted: Boolean? = null,
) : KafkatorioKeyedPacketData(), MapChunkUpdateKey {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.MAP_CHUNK
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
sealed interface PlayerUpdateKey : KafkatorioKeyedPacketKey {
val index: PlayerIndex
}
@Serializable
data class PlayerUpdate(
override val index: PlayerIndex,
override val eventCounts: Map<String, UInt>? = null,
val characterUnitNumber: UnitNumber? = null,
val chatColour: Colour? = null,
val colour: Colour? = null,
val name: String? = null,
val afkTime: Tick? = null,
val ticksToRespawn: Tick? = null,
val forceIndex: ForceIndex? = null,
val isAdmin: Boolean? = null,
val isConnected: Boolean? = null,
val isShowOnMap: Boolean? = null,
val isSpectator: Boolean? = null,
val lastOnline: Tick? = null,
val onlineTime: Tick? = null,
val position: MapEntityPosition? = null,
val surfaceIndex: SurfaceIndex? = null,
val tag: String? = null,
val diedCause: EntityIdentifiersData? = null,
val bannedReason: String? = null,
val kickedReason: String? = null,
val disconnectReason: String? = null,
/** `true` when a player is removed (deleted) from the game */
val isRemoved: Boolean? = null,
) : KafkatorioKeyedPacketData(), PlayerUpdateKey {
@EncodeDefault
override val updateType: KafkatorioPacketDataType = KafkatorioPacketDataType.PLAYER
}
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */

33
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/json.kt → modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/json.kt
View file

@ -1,13 +1,19 @@
package dev.adamko.kafkatorio.events.schema
package dev.adamko.kafkatorio.schema
import kotlin.reflect.KClass
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.KSerializer
import kotlinx.serialization.builtins.ListSerializer
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonContentPolymorphicSerializer
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonTransformingSerializer
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import kotlinx.serialization.modules.SerializersModule
@ -44,3 +50,28 @@ class FactorioJsonListSerializer<T>(dataSerializer: KSerializer<T>) :
}
}
interface SerializerProvider<T : Any> {
val serializer: KSerializer<out T>
}
open class SerializerProviderDeserializer<T : Any>(
private val baseClass: KClass<T>,
private val key: String,
private val selector: (String) -> SerializerProvider<out T>?,
) : JsonContentPolymorphicSerializer<T>(baseClass) {
override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out T> {
val type = element
.jsonObject[key]
?.jsonPrimitive
?.contentOrNull
?.let { json -> selector(json) }
requireNotNull(type) { "Unknown ${baseClass.simpleName} $key: $element" }
return type.serializer
}
}

49
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/events/schema/factorioPrototypes.kt → modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/prototypes/factorioPrototypes.kt
View file

@ -1,9 +1,14 @@
package dev.adamko.kafkatorio.events.schema
package dev.adamko.kafkatorio.schema.prototypes
import dev.adamko.kafkatorio.events.schema.FactorioPrototype.PrototypeObjectName
import dev.adamko.kafkatorio.schema.SerializerProvider
import dev.adamko.kafkatorio.schema.SerializerProviderDeserializer
import dev.adamko.kafkatorio.schema.common.Colour
import dev.adamko.kafkatorio.schema.common.PrototypeName
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype.PrototypeObjectName
import kotlinx.serialization.Contextual
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.EncodeDefault
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonContentPolymorphicSerializer
import kotlinx.serialization.json.JsonElement
@ -12,46 +17,30 @@ import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
@Serializable(with = FactorioPrototypeJsonSerializer::class)
@Serializable(with = FactorioPrototype.JsonSerializer::class)
sealed class FactorioPrototype {
@EncodeDefault
abstract val prototypeObjectName: PrototypeObjectName
enum class PrototypeObjectName {
LuaTilePrototype,
enum class PrototypeObjectName(
override val serializer: KSerializer<out FactorioPrototype>
) : SerializerProvider<FactorioPrototype> {
LuaTilePrototype(MapTilePrototype.serializer()),
;
companion object {
val values: List<PrototypeObjectName> = values().toList()
val entries: Set<PrototypeObjectName> = values().toSet()
}
}
object JsonSerializer : SerializerProviderDeserializer<FactorioPrototype>(
FactorioPrototype::class,
FactorioPrototype::prototypeObjectName.name,
{ json -> PrototypeObjectName.entries.firstOrNull { it.name == json } }
)
}
object FactorioPrototypeJsonSerializer : JsonContentPolymorphicSerializer<FactorioPrototype>(
FactorioPrototype::class
) {
private val key = FactorioPrototype::prototypeObjectName.name
override fun selectDeserializer(element: JsonElement): DeserializationStrategy<out FactorioPrototype> {
val type = element
.jsonObject[key]
?.jsonPrimitive
?.contentOrNull
?.let { json ->
PrototypeObjectName.values.firstOrNull { it.name == json }
}
requireNotNull(type) { "Unknown FactorioPrototype ${key}: $element" }
return when (type) {
PrototypeObjectName.LuaTilePrototype -> MapTilePrototype.serializer()
}
}
}
@Serializable
data class MapTilePrototype(

23
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/prototypes/factorioPrototypes2.kt Normal file
View file

@ -0,0 +1,23 @@
package dev.adamko.kafkatorio.schema.prototypes
import dev.adamko.kafkatorio.schema.common.Colour
import dev.adamko.kafkatorio.schema.common.PrototypeName
import kotlinx.serialization.Contextual
import kotlinx.serialization.Serializable
@Serializable
sealed class FactorioPrototype2 {
@Serializable
data class MapTile(
val name: PrototypeName,
val layer: UInt,
val mapColour: Colour,
@Contextual
val collisionMasks: List<String>,
val order: String,
/** Can the tile be mined for resources? */
val canBeMined: Boolean,
) : FactorioPrototype2()
}

16
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema2/kafkatorioPacket2.kt Normal file
View file

@ -0,0 +1,16 @@
package dev.adamko.kafkatorio.schema2
import dev.adamko.kafkatorio.schema.common.Tick
import kotlinx.serialization.Serializable
@Serializable
data class KafkatorioPacket2(
val modVersion: String,
val tick: Tick,
val data: KafkatorioPacketData2,
)
@Serializable
sealed class KafkatorioPacketData2

70
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema2/kafkatorioPacketInstant2.kt Normal file
View file

@ -0,0 +1,70 @@
package dev.adamko.kafkatorio.schema2
import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
import kotlinx.serialization.Serializable
/** No key, no caching. Must be sent instantly. */
@Serializable
sealed class KafkatorioInstantPacketData2 : KafkatorioPacketData2()
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class ConfigurationUpdate(
val factorioData: ConfigurationUpdateGameData,
val allMods: List<ConfigurationUpdateModData>,
val modStartupSettingsChange: Boolean,
val migrationApplied: Boolean,
) : KafkatorioInstantPacketData2() {
@Serializable
data class ConfigurationUpdateGameData(
val oldVersion: String? = null,
val newVersion: String? = null,
)
@Serializable
data class ConfigurationUpdateModData(
val modName: String,
val currentVersion: String? = null,
val previousVersion: String? = null,
)
}
@Serializable
data class ConsoleChatUpdate(
val authorPlayerIndex: PlayerIndex?,
val content: String,
) : KafkatorioInstantPacketData2()
@Serializable
data class ConsoleCommandUpdate(
val authorPlayerIndex: PlayerIndex?,
val command: String,
val parameters: String,
) : KafkatorioInstantPacketData2()
@Serializable
data class PrototypesUpdate(
val prototypes: List<FactorioPrototype2>,
) : KafkatorioInstantPacketData2()
@Serializable
data class SurfaceUpdate(
val index: SurfaceIndex,
val daytime: Double,
val name: String,
) : KafkatorioInstantPacketData2()
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */

126
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema2/kafkatorioPacketKeyed2.kt Normal file
View file

@ -0,0 +1,126 @@
package dev.adamko.kafkatorio.schema2
import dev.adamko.kafkatorio.schema.common.Colour
import dev.adamko.kafkatorio.schema.common.EntityIdentifiers
import dev.adamko.kafkatorio.schema.common.EntityIdentifiersData
import dev.adamko.kafkatorio.schema.common.ForceIndex
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapEntityPosition
import dev.adamko.kafkatorio.schema.common.MapTileDictionary
import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.PrototypeName
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.common.UnitNumber
import kotlinx.serialization.Serializable
/** Has a key, so it can be debounced */
@Serializable
sealed class KafkatorioKeyedPacketData2 : KafkatorioPacketData2() {
/** Count how many events this packet is aggregating data from */
abstract val key: KafkatorioKeyedPacketKey2
abstract val eventCounts: Map<String, UInt>?
}
@Serializable
sealed class KafkatorioKeyedPacketKey2
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class EntityUpdateKey(
/** While normally optional, here a [unitNumber] is required for caching */
override val unitNumber: UnitNumber,
override val name: String,
override val protoType: String,
) : EntityIdentifiers, KafkatorioKeyedPacketKey2()
@Serializable
data class EntityUpdate(
override val key: EntityUpdateKey,
override val eventCounts: Map<String, UInt>? = null,
val chunkPosition: MapChunkPosition? = null,
val graphicsVariation: UByte? = null,
val health: Float? = null,
val isActive: Boolean? = null,
val isRotatable: Boolean? = null,
val lastUser: UInt? = null,
val localisedDescription: String? = null,
val localisedName: String? = null,
val prototype: PrototypeName? = null,
) : KafkatorioKeyedPacketData2()
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class MapChunkUpdateKey(
val chunkPosition: MapChunkPosition,
val surfaceIndex: SurfaceIndex,
) : KafkatorioKeyedPacketKey2()
@Serializable
data class MapChunkUpdate(
override val key: MapChunkUpdateKey,
override val eventCounts: Map<String, UInt>? = null,
val player: PlayerIndex? = null,
val robot: EntityIdentifiersData? = null,
val force: ForceIndex? = null,
/** updated tiles - might be partial and not all tiles in the chunk */
val tileDictionary: MapTileDictionary? = null,
val isDeleted: Boolean? = null,
) : KafkatorioKeyedPacketData2()
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */
@Serializable
data class PlayerUpdateKey(
val index: PlayerIndex,
) : KafkatorioKeyedPacketKey2()
@Serializable
data class PlayerUpdate(
override val key: PlayerUpdateKey,
override val eventCounts: Map<String, UInt>? = null,
val characterUnitNumber: UnitNumber? = null,
val chatColour: Colour? = null,
val colour: Colour? = null,
val name: String? = null,
val afkTime: Tick? = null,
val ticksToRespawn: Tick? = null,
val forceIndex: ForceIndex? = null,
val isAdmin: Boolean? = null,
val isConnected: Boolean? = null,
val isShowOnMap: Boolean? = null,
val isSpectator: Boolean? = null,
val lastOnline: Tick? = null,
val onlineTime: Tick? = null,
val position: MapEntityPosition? = null,
val surfaceIndex: SurfaceIndex? = null,
val tag: String? = null,
val diedCause: EntityIdentifiersData? = null,
val bannedReason: String? = null,
val kickedReason: String? = null,
val disconnectReason: String? = null,
/** `true` when a player is removed (deleted) from the game */
val isRemoved: Boolean? = null,
) : KafkatorioKeyedPacketData2()
/* *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** */

9
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/ColourHexTest.kt
View file

@ -1,7 +1,10 @@
package dev.adamko.kafkatorio.events.schema
import dev.adamko.kafkatorio.events.schema.converters.toHex
import dev.adamko.kafkatorio.events.schema.converters.toPercentile
import dev.adamko.kafkatorio.schema.common.Colour
import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.toHex
import dev.adamko.kafkatorio.schema.common.toPercentile
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.BehaviorSpec
import io.kotest.matchers.floats.shouldBeWithinPercentageOf
@ -81,6 +84,4 @@ class ColourHexTest : BehaviorSpec({
}
}
})

342
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/FactorioEventTest.kt
View file

@ -1,342 +0,0 @@
package dev.adamko.kafkatorio.events.schema
import io.kotest.assertions.json.shouldEqualJson
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
class FactorioEventTest : FunSpec({
// context("Given: json on_player_joined_game") {
//
// // language=JSON
// val json = """
// {
// "data": {
// "objectName": "LuaPlayer",
// "force": 1,
// "surface": 1,
// "name": "fredthedeadhead",
// "characterUnitNumber": 1,
// "associatedCharactersUnitNumbers": {},
// "position": {
// "x": 30.6,
// "y": 84.1
// },
// "colour": {
// "red": 0.86,
// "green": 0.5,
// "blue": 0.12,
// "alpha": 0.5
// },
// "chatColour": {
// "red": 1,
// "green": 0.62,
// "blue": 0.25
// },
// "lastOnline": 2287061
// },
// "packetType": "EVENT",
// "eventType": "on_player_joined_game",
// "modVersion": "0.2.1",
// "tick": 2287072
// }
// """.trimIndent()
// context("When: decoded") {
// val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
//
// test("Then: parse") {
//
// val expected = FactorioEvent(
// data = PlayerData(
// name = "fredthedeadhead",
// surface = SurfaceIndex(1u),
// force = ForceIndex(1u),
// characterUnitNumber = UnitNumber(1u),
// associatedCharactersUnitNumbers = emptyList(),
// position = MapEntityPosition(
// x = 30.6,
// y = 84.1,
// ),
// colour = Colour(
// red = 0.86f,
// green = 0.5f,
// blue = 0.12f,
// alpha = 0.5f
// ),
// chatColour = Colour(
// red = 1f,
// green = 0.62f,
// blue = 0.25f,
// alpha = 1f
// ),
// lastOnline = Tick(2287061u)
// ),
// eventType = "on_player_joined_game",
// modVersion = "0.2.1",
// tick = Tick(2287072u)
// )
//
// actual shouldBe expected
//
// expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaPlayer
// }
//
// test("Then: expect encode equals json") {
// val encoded = jsonMapperKafkatorio.encodeToString(actual)
// encoded.shouldEqualJson(json)
// }
// }
// }
//
// context("Given: json LuaSurface") {
// // language=JSON
// val json = """
// {
// "data": {
// "objectName": "LuaSurface",
// "name": "nauvis",
// "index": 1,
// "daytime": 0.745
// },
// "packetType": "EVENT",
// "eventType": "on_tick",
// "modVersion": "0.2.1",
// "tick": 2301240
// }
// """.trimIndent()
//
// context("When: decoded") {
// val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
//
// test("should equal obj") {
//
// val expected = FactorioEvent(
// data = SurfaceData(
// name = "nauvis",
// index = SurfaceIndex(1u),
// daytime = 0.745
// ),
// eventType = "on_tick",
// modVersion = "0.2.1",
// tick = Tick(2301240u)
// )
//
// actual shouldBe expected
//
// expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaSurface
// }
//
// test("Then: expect encode equals json") {
// val encoded = jsonMapperKafkatorio.encodeToString(actual)
// encoded.shouldEqualJson(json)
// }
// }
// }
//
// context("Given: LuaEntity on_player_changed_position") {
// // language=JSON
// val json = """
// {
// "data": {
// "objectName": "LuaEntity",
// "name": "character",
// "type": "character",
// "active": true,
// "health": 250,
// "healthRatio": 1,
// "surface": 1,
// "force": 1,
// "unitNumber": 1,
// "position": {
// "x": 37.5,
// "y": 81.1
// }
// },
// "packetType": "EVENT",
// "eventType": "on_player_changed_position",
// "modVersion": "0.2.1",
// "tick": 2301231
// }
// """.trimIndent()
// context("When: decoded") {
// val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
//
// test("parse") {
//
// val expected = FactorioEvent(
// data = EntityData(
// name = "character",
// type = "character",
// active = true,
// health = 250.0,
// healthRatio = 1.0,
// surface = SurfaceIndex(1u),
// unitNumber = UnitNumber(1u),
// force = ForceIndex(1u),
// playerIndex = null,
// position = MapEntityPosition(
// x = 37.5,
// y = 81.1,
// )
// ),
// eventType = "on_player_changed_position",
// modVersion = "0.2.1",
// tick = Tick(2301231u)
// )
//
// actual shouldBe expected
//
// expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaEntity
// }
//
// test("Then: expect encode equals json") {
// val encoded = jsonMapperKafkatorio.encodeToString(actual)
// encoded.shouldEqualJson(json)
// }
// }
//
// }
//
// context("Given: LuaTiles on_player_built_tile") {
// // language=JSON
// val json = """
// {
// "data": {
// "objectName": "LuaTiles",
// "surfaceIndex": 1,
// "tiles": [
// {
// "y": 10,
// "x": -65,
// "proto": "refined-concrete"
// },
// {
// "y": 11,
// "x": -65,
// "proto": "red-earth"
// },
// {
// "y": 10,
// "x": -64,
// "proto": "blue-cheese"
// },
// {
// "y": 11,
// "x": -64,
// "proto": "refined-concrete"
// }
// ]
// },
// "packetType": "EVENT",
// "eventType": "on_player_built_tile",
// "modVersion": "0.2.2",
// "tick": 2429929
// }
// """.trimIndent()
// context("When: decoded") {
// val actual: KafkatorioPacket = jsonMapperKafkatorio.decodeFromString(json)
//
// test("parse") {
//
// val expected = FactorioEvent(
// data = MapTiles(
// surfaceIndex = SurfaceIndex(1u),
// tiles = listOf(
// MapTile(
// proto = PrototypeName("refined-concrete"),
// x = -65,
// y = 10,
// ),
// MapTile(
// proto = PrototypeName("red-earth"),
// x = -65,
// y = 11,
// ),
// MapTile(
// proto = PrototypeName("blue-cheese"),
// x = -64,
// y = 10,
// ),
// MapTile(
// proto = PrototypeName("refined-concrete"),
// x = -64,
// y = 11,
// ),
// )
// ),
// eventType = "on_player_built_tile",
// modVersion = "0.2.2",
// tick = Tick(2429929u)
// )
//
// actual shouldBe expected
//
// expected.data.objectName shouldBe FactorioObjectData.ObjectName.LuaTiles
// }
//
// test("Then: expect encode equals json") {
// val encoded = jsonMapperKafkatorio.encodeToString(actual)
// encoded.shouldEqualJson(json)
// }
// }
// }
//
// context("Given: packet FactorioConfigurationUpdate") {
// // language=JSON
// val json = """
// {
// "modVersion": "0.2.8",
// "packetType": "CONFIG",
// "tick": 534,
// "allMods": [
// {
// "modName": "kafkatorio-events",
// "currentVersion": "0.2.8",
// "previousVersion": "0.2.7"
// },
// {
// "modName": "base",
// "currentVersion": "1.1.53"
// }
// ],
// "factorioData": {}
// }
// """.trimIndent()
// context("When: decoded") {
// val actual: FactorioConfigurationUpdate = jsonMapperKafkatorio.decodeFromString(json)
//
// test("parse") {
//
// val expected = FactorioConfigurationUpdate(
// modVersion = "0.2.8",
// tick = Tick(534u),
// factorioData = FactorioGameDataUpdate(null, null),
// allMods = listOf(
// FactorioModInfo(
// modName = "kafkatorio-events",
// currentVersion = "0.2.8",
// previousVersion = "0.2.7"
// ),
// FactorioModInfo(
// modName = "base",
// currentVersion = "1.1.53",
// previousVersion = null
// ),
// )
// )
//
// actual shouldBe expected
//
// expected.packetType shouldBe KafkatorioPacket.PacketType.CONFIG
// }
//
// test("Then: expect encode equals json") {
// val encoded = jsonMapperKafkatorio.encodeToString(actual)
// encoded.shouldEqualJson(json)
// }
// }
//
// }
})

93
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/events/schema/MapChunkUpdateTest.kt Normal file
View file

@ -0,0 +1,93 @@
package dev.adamko.kafkatorio.events.schema
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapTileDictionary
import dev.adamko.kafkatorio.schema.common.PrototypeName
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.MapChunkUpdateKey
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe
import kotlin.math.exp
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
class MapChunkUpdateTest : FunSpec({
context("MapChunkUpdate update") {
//language=json
val json = """
{
"data": {
"type": "dev.adamko.kafkatorio.schema2.MapChunkUpdate",
"key": {
"surfaceIndex": 1,
"chunkPosition": [
-6,
-3
]
},
"tileDictionary": {
"tilesXY": {
"-192": {
"-65": 2
},
"161": {
"-96": 1,
"-95": 2,
"94": 2,
"93": 32
}
},
"protos": {
"water": 1,
"grass-4": 2,
"grass-3": 32
}
},
"eventCounts": {
"on_chunk_generated": 1
}
},
"modVersion": "0.4.0",
"tick": 36
}
""".trimIndent()
val expected = KafkatorioPacket2(
tick = Tick(36u),
modVersion = "0.4.0",
data = MapChunkUpdate(
key = MapChunkUpdateKey(
MapChunkPosition(-6, -3),
SurfaceIndex(1u),
),
eventCounts = mapOf("on_chunk_generated" to 1u),
tileDictionary = MapTileDictionary(
tilesXY = mapOf(
"-192" to mapOf("-65" to MapTileDictionary.PrototypeKey(2)),
"161" to mapOf(
"-96" to MapTileDictionary.PrototypeKey(1),
"-95" to MapTileDictionary.PrototypeKey(2),
"94" to MapTileDictionary.PrototypeKey(2),
"93" to MapTileDictionary.PrototypeKey(32),
),
),
protos = mapOf(
PrototypeName("water") to MapTileDictionary.PrototypeKey(1),
PrototypeName("grass-4") to MapTileDictionary.PrototypeKey(2),
PrototypeName("grass-3") to MapTileDictionary.PrototypeKey(32),
)
)
)
)
test("decode") {
val packet: KafkatorioPacket2 = jsonMapperKafkatorio.decodeFromString(json)
packet shouldBe expected
}
}
})

42
modules/events-schema/src/commonTest/kotlin/dev/adamko/kafkatorio/schema/common/MapTileDictionaryTest.kt Normal file
View file

@ -0,0 +1,42 @@
package dev.adamko.kafkatorio.schema.common
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.collections.shouldContainAll
class MapTileDictionaryTest : FunSpec({
context("MapTileDictionary") {
val actual = MapTileDictionary(
tilesXY = mapOf(
"-192" to mapOf("-65" to MapTileDictionary.PrototypeKey(2)),
"161" to mapOf(
"-96" to MapTileDictionary.PrototypeKey(1),
"-95" to MapTileDictionary.PrototypeKey(2),
"94" to MapTileDictionary.PrototypeKey(2),
"93" to MapTileDictionary.PrototypeKey(32),
),
// unknown prototype
"44" to mapOf("1" to MapTileDictionary.PrototypeKey(44)),
),
protos = mapOf(
PrototypeName("water") to MapTileDictionary.PrototypeKey(1),
PrototypeName("grass-4") to MapTileDictionary.PrototypeKey(2),
PrototypeName("grass-3") to MapTileDictionary.PrototypeKey(32),
PrototypeName("not-used") to MapTileDictionary.PrototypeKey(99),
)
)
test("toMapTileList") {
actual.toMapTileList() shouldContainAll listOf(
MapTile(-192, -65, PrototypeName("grass-4")),
MapTile(161, -96, PrototypeName("water")),
MapTile(161, -95, PrototypeName("grass-4")),
MapTile(161, 94, PrototypeName("grass-4")),
MapTile(161, 93, PrototypeName("grass-3")),
)
}
}
})

93
modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts.kt
View file

@ -1,5 +1,10 @@
package dev.adamko.kafkatorio.events.schema
import dev.adamko.kafkatorio.schema.events.KafkatorioInstantPacketData
import dev.adamko.kafkatorio.schema.events.KafkatorioKeyedPacketData
import dev.adamko.kafkatorio.schema.events.KafkatorioKeyedPacketKey
//import dev.adamko.kafkatorio.schema.events.KafkatorioPacket
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype
import kotlin.io.path.Path
import kotlin.io.path.createDirectories
import me.ntrrgc.tsGenerator.TypeScriptGenerator
@ -10,22 +15,20 @@ import org.intellij.lang.annotations.Language
*/
fun main(args: Array<String>) {
val gen =
TypeScriptGenerator(
val gen = TypeScriptGenerator(
rootClasses = buildSet {
addAll(KafkatorioPacket::class.sealedSubclasses)
// add(KafkatorioPacket::class)
// addAll(KafkatorioPacket::class.sealedSubclasses)
addAll(FactorioObjectData::class.sealedSubclasses)
add(KafkatorioInstantPacketData::class)
addAll(KafkatorioInstantPacketData::class.sealedSubclasses)
add(KafkatorioKeyedPacketData::class)
addAll(KafkatorioKeyedPacketData::class.sealedSubclasses)
addAll(KafkatorioKeyedPacketKey::class.sealedSubclasses)
add(FactorioPrototype::class)
addAll(FactorioPrototype::class.sealedSubclasses)
add(FactorioPrototypes::class)
add(FactorioConfigurationUpdate::class)
add(FactorioEvent::class)
add(FactorioEventUpdatePacket::class)
addAll(FactorioEventUpdate::class.sealedSubclasses)
addAll(FactorioEventUpdateKey::class.sealedSubclasses)
},
mappings = mapOf(
// builtin Factorio numeric types > `typed-factorio/generated/builtin-types.d.ts`
@ -57,7 +60,7 @@ fun main(args: Array<String>) {
.forEach { (defFile, def) ->
val outFile = outPath.resolve(defFile).toFile()
outFile.printWriter().use {
it.println(header)
it.println(kt2tsHeader)
it.print(def)
}
println("Generated file: $outFile")
@ -67,6 +70,7 @@ fun main(args: Array<String>) {
}
}
/** Split the TypeScript interfaces into different files */
private fun splitDefinitions(definitions: String): Map<String, String> {
return definitions
@ -74,51 +78,64 @@ private fun splitDefinitions(definitions: String): Map<String, String> {
.groupBy { def ->
when {
def isInterface "KafkatorioPacket"
|| def extendsAny listOf("KafkatorioPacket")
|| def isType "KafkatorioPacketDataType"
-> "kafkatorio-packet.d.ts"
"interface FactorioObjectData " in def ||
"type ObjectName " in def ||
def.contains(Regex("interface.+extends.+(FactorioObjectData).*"))
-> "object-data.d.ts"
def isInterface "KafkatorioInstantPacketData"
|| def extends "KafkatorioInstantPacketData"
|| def isType "KafkatorioInstantPacketDataType"
"extends FactorioPrototype " in def ||
"interface FactorioPrototypes " in def ||
"interface FactorioPrototype " in def ||
"type PrototypeObjectName " in def -> "prototype.d.ts"
|| def isInterface "ConfigurationUpdateGameData"
|| def isInterface "ConfigurationUpdateModData"
-> "kafkatorio-instant-packet.d.ts"
"extends FactorioConfigurationUpdate " in def ||
"interface FactorioConfigurationUpdate " in def ||
"FactorioGameDataUpdate " in def ||
"FactorioModInfo " in def -> "config-update.d.ts"
def isInterface "KafkatorioKeyedPacket"
|| def isInterface "KafkatorioKeyedPacketData"
|| def extends "KafkatorioKeyedPacket"
|| def isType "KafkatorioKeyedPacketDataType"
"interface FactorioEventUpdatePacket" in def ||
"type FactorioEventUpdateType =" in def ||
"interface FactorioEventUpdateKey {" in def ||
"interface FactorioEventUpdateData {" in def ||
def.contains(
Regex(
"""
interface.+extends.+(FactorioEventUpdate|PlayerUpdateKey|PlayerUpdateData).*
""".trimIndent()
)
)
-> "update.d.ts"
|| def isInterface "KafkatorioKeyedPacketKey"
-> "kafkatorio-keyed-packet.d.ts"
def isInterface "FactorioPrototypes"
|| def isInterface "FactorioPrototype"
|| def extends "FactorioPrototype"
|| def isType "PrototypeObjectName"
-> "prototype.d.ts"
else -> "schema.d.ts"
}
}
.mapValues { (_, def) ->
def
.sorted()
.joinToString("\n\n", postfix = "\n")
.replace(" ", " ")
}
}
@Language("TypeScript")
val header = """
val kt2tsHeader = """
|// Generated by TypeScriptGenerator - do not edit this file manually
|
""".trimMargin()
private infix fun String.extends(types: String): Boolean =
contains(Regex(".+extends.*(${types})"))
private infix fun String.extendsAny(types: List<String>): Boolean =
extends(types.joinToString("|") { "$it,".replaceAfterLast(',', "") })
private infix fun String.isInterface(type: String): Boolean =
contains(Regex("interface $type.*"))
private infix fun String.isType(type: String): Boolean =
contains(Regex("type $type.*"))
//object PropertyNameTransformer : ClassTransformer {
// override fun transformPropertyName(
// propertyName: String,

55
modules/events-schema/src/jvmMain/kotlin/dev/adamko/kafkatorio/events/schema/kt2ts2.kt Normal file
View file

@ -0,0 +1,55 @@
package dev.adamko.kafkatorio.events.schema
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kxstsgen.KxsTsConfig
import dev.adamko.kxstsgen.KxsTsGenerator
import dev.adamko.kxstsgen.core.TsDeclaration.TsTypeAlias
import dev.adamko.kxstsgen.core.TsElementId
import dev.adamko.kxstsgen.core.TsLiteral
import dev.adamko.kxstsgen.core.TsTypeRef
import kotlin.io.path.Path
import kotlin.io.path.createDirectories
import kotlinx.serialization.builtins.serializer
@ExperimentalUnsignedTypes
fun main(args: Array<String>) {
val config = KxsTsConfig(
// typeAliasTyping = BrandTyping
)
val gen2 = KxsTsGenerator(config)
gen2.descriptorOverrides += mapOf(
//@formatter:off
Float .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Float" ), TsTypeRef.Literal(TsLiteral.Custom("float" ), false)),
Double.serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Double"), TsTypeRef.Literal(TsLiteral.Custom("double"), false)),
Int .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Int" ), TsTypeRef.Literal(TsLiteral.Custom("int" ), false)),
Byte .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.Byte" ), TsTypeRef.Literal(TsLiteral.Custom("int8" ), false)),
UInt .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.UInt" ), TsTypeRef.Literal(TsLiteral.Custom("uint" ), false)),
UByte .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.UByte" ), TsTypeRef.Literal(TsLiteral.Custom("uint8" ), false)),
UShort.serializer().descriptor to TsTypeAlias(TsElementId("kotlin.UShort"), TsTypeRef.Literal(TsLiteral.Custom("uint16"), false)),
ULong .serializer().descriptor to TsTypeAlias(TsElementId("kotlin.ULong" ), TsTypeRef.Literal(TsLiteral.Custom("uint64"), false)),
//@formatter:on
)
val definitions = gen2.generate(
KafkatorioPacket2.serializer(),
// KafkatorioKeyedPacketKey2.serializer(),
)
when (val filename = args.firstOrNull()) {
null -> println(definitions)
else -> {
val outPath = Path(filename)
outPath.createDirectories()
val outFile = outPath.resolve("kafkatorio-schema.ts").toFile()
outFile.printWriter().use {
it.println(kt2tsHeader)
it.print(definitions)
}
}
}
}

3
modules/events-server/build.gradle.kts
View file

@ -41,6 +41,9 @@ dependencies {
implementation(libs.scrimage.core)
implementation(platform(libs.okio.bom))
implementation(libs.okio.core)
implementation(libs.kafka.kotkaStreams) {
isChanging = true
}

23
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt
View file

@ -7,9 +7,9 @@ import dev.adamko.kafkatorio.processor.topology.playerUpdatesToWsServer
import dev.adamko.kafkatorio.processor.topology.saveMapTiles
import dev.adamko.kafkatorio.processor.topology.splitFactorioServerPacketStream
import java.time.Duration
import java.util.Properties
import kotlin.coroutines.CoroutineContext
import kotlin.coroutines.cancellation.CancellationException
import kotlin.time.Duration.Companion.minutes
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@ -31,15 +31,12 @@ internal class KafkatorioTopology(
override val coroutineContext: CoroutineContext =
Dispatchers.Default + SupervisorJob() + CoroutineName("KafkatorioTopology")
companion object {
const val TOPIC_SRC_SERVER_LOG = "factorio-server-log"
const val TOPIC_GROUPED_MAP_CHUNKS = "kafkatorio.MAP.grouped-map-chunks"
}
fun start() {
splitPackets()
groupTilesMapChunks()
saveTiles()
splitPackets()
playerUpdates()
}
@ -56,7 +53,6 @@ internal class KafkatorioTopology(
}
private fun splitPackets() {
val builder = StreamsBuilder()
val packets = factorioServerPacketStream(builder)
splitFactorioServerPacketStream(packets)
@ -75,12 +71,19 @@ internal class KafkatorioTopology(
launchTopology("playerUpdates", topology)
}
private fun launchTopology(id: String, topology: Topology) {
private fun launchTopology(
id: String,
topology: Topology,
config: Map<String, String> = mapOf()
) {
val props = appProps.kafkaConfig
val props: Properties = appProps.kafkaConfig
config.forEach { (k, v) -> props.setProperty(k, v) }
val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as? String
props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
// props.setProperty(StreamsConfig.MAX_TASK_IDLE_MS_CONFIG, "${30.minutes.inWholeMilliseconds}")
val streams = KafkaStreams(topology, props)

1
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/StreamsExceptionHandler.kt
View file

@ -3,6 +3,7 @@ package dev.adamko.kafkatorio.processor
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse
class StreamsExceptionHandler(
val shutdown: (e: Throwable) -> Unit,
) : StreamsUncaughtExceptionHandler {

42
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/KafkatorioKafkaAdmin.kt
View file

@ -1,9 +1,5 @@
package dev.adamko.kafkatorio.processor.admin
import dev.adamko.kafkatorio.events.schema.FactorioEventUpdate
import dev.adamko.kafkatorio.events.schema.FactorioObjectData
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
import dev.adamko.kafkatorio.processor.KafkatorioTopology
import java.util.concurrent.TimeUnit
import mu.KotlinLogging
import org.apache.kafka.clients.admin.Admin
@ -28,34 +24,9 @@ object KafkatorioKafkaAdmin {
val currentTopics = currentTopics().map { it.name() }
val kafkatorioTopics = buildSet {
KafkatorioPacket.PacketType.values.forEach { packetType ->
add(KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS)
add(KafkatorioTopology.TOPIC_SRC_SERVER_LOG)
val base = topicNameBase(packetType)
when (packetType) {
KafkatorioPacket.PacketType.EVENT ->
FactorioObjectData.ObjectName.values.forEach { objectName ->
add("$base.${objectName.name}")
}
KafkatorioPacket.PacketType.CONFIG -> add("$base.FactorioConfigurationUpdate")
KafkatorioPacket.PacketType.PROTOTYPES -> add("$base.all")
KafkatorioPacket.PacketType.UPDATE -> {
add("$base.all")
FactorioEventUpdate.FactorioEventUpdateType.values.forEach { objectName ->
add("$base.${objectName.name}")
}
}
}
}
}.minus(currentTopics.toSet())
val kafkatorioTopics = allTopics() - currentTopics.toSet()
if (kafkatorioTopics.isNotEmpty()) {
logger.info("Creating ${kafkatorioTopics.size} topics")
val result = kafkaAdmin.createTopics { kafkatorioTopics }
@ -83,11 +54,14 @@ object KafkatorioKafkaAdmin {
replicationFactor: Short = 1,
topicNames: () -> Collection<String>,
): CreateTopicsResult =
createTopics(topicNames()
createTopics(
topicNames()
.distinct()
.map {
NewTopic(it, numPartitions, replicationFactor)
})
// .configs(mapOf("cleanup.policy" to "compact"))
}
)
private fun currentTopics(): MutableCollection<TopicListing> {
return kafkaAdmin.listTopics()
@ -95,6 +69,6 @@ object KafkatorioKafkaAdmin {
.get()
}
fun topicNameBase(packetType: KafkatorioPacket.PacketType): String =
"kafkatorio.${packetType.name}"
// fun topicNameBase(packetType: KafkatorioPacketDataType): String =
// "kafkatorio.${packetType.name}"
}

104
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt Normal file
View file

@ -0,0 +1,104 @@
package dev.adamko.kafkatorio.processor.admin
import dev.adamko.kafkatorio.schema2.ConfigurationUpdate
import dev.adamko.kafkatorio.schema2.ConsoleChatUpdate
import dev.adamko.kafkatorio.schema2.ConsoleCommandUpdate
import dev.adamko.kafkatorio.schema2.EntityUpdate
import dev.adamko.kafkatorio.schema2.KafkatorioPacketData2
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.PlayerUpdate
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kafkatorio.schema2.SurfaceUpdate
import kotlin.reflect.KClass
private const val DOMAIN = "kafkatorio"
const val TOPIC_SRC_SERVER_LOG = "$DOMAIN.src.server-log"
// "factorio-server-log"
const val TOPIC_GROUPED_MAP_CHUNKS_STATE = "$DOMAIN.state.map-chunks.grouped"
fun allTopics(): Set<String> = buildSet {
add(TOPIC_SRC_SERVER_LOG)
add(TOPIC_GROUPED_MAP_CHUNKS_STATE)
addAll(packetTopicNames.values)
}
//fun KafkatorioPacket2.topicName(): String = data.topicName()
//fun KafkatorioPacketData2.topicName(): String = "$DOMAIN.packet.${topicName}"
val KafkatorioPacketData2.topicName: String
get() = when (this) {
is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class)
is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class)
is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class)
is PrototypesUpdate -> packetTopicNames.getValue(PrototypesUpdate::class)
is SurfaceUpdate -> packetTopicNames.getValue(SurfaceUpdate::class)
is EntityUpdate -> packetTopicNames.getValue(EntityUpdate::class)
is MapChunkUpdate -> packetTopicNames.getValue(MapChunkUpdate::class)
is PlayerUpdate -> packetTopicNames.getValue(PlayerUpdate::class)
}
private val packetTopicNames: Map<KClass<out KafkatorioPacketData2>, String> = mapOf(
ConfigurationUpdate::class to "configuration",
ConsoleChatUpdate::class to "console-chat",
ConsoleCommandUpdate::class to "console-command",
PrototypesUpdate::class to "prototypes",
SurfaceUpdate::class to "surface",
EntityUpdate::class to "entity",
MapChunkUpdate::class to "map-chunk",
PlayerUpdate::class to "player",
).mapValues { (_, v) ->
"$DOMAIN.packet.$v"
}
val KClass<out KafkatorioPacketData2>.topicName: String
get() = packetTopicNames.getValue(this)
//fun packetTopic(
// dataType: KafkatorioKeyedPacketData.KafkatorioKeyedPacketDataType,
//): String = packetTopic(KafkatorioPacketType.KEYED, dataType.name.lowercase())
//
//private fun packetTopic(
// packetType: KafkatorioPacketType,
// dataType: String,
//): String {
// return "$DOMAIN.packet.$packetType.$dataType"
//}
//sealed interface KafkatorioTopic {
//
// val type: String
// val entity: String
// val description: String
//
// companion object {
// private const val DOMAIN = "kafkatorio"
//
// val KafkatorioTopic.topic
// get() = listOf(DOMAIN, type, entity, description).joinToString(".")
//
// }
//
// data class Event(
// override val entity: String,
// override val description: String,
// ) : KafkatorioTopic {
// override val type: String = "event"
// }
//
//
// data class State(
// override val entity: String,
// override val description: String,
// ) : KafkatorioTopic {
// override val type: String = "state"
// }
//
//}

67
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes/kxsBinary.kt
View file

@ -1,11 +1,5 @@
package dev.adamko.kafkatorio.processor.serdes
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.DataInput
import java.io.DataInputStream
import java.io.DataOutput
import java.io.DataOutputStream
import kotlinx.serialization.BinaryFormat
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.SerializationStrategy
@ -17,6 +11,9 @@ import kotlinx.serialization.encoding.CompositeDecoder
import kotlinx.serialization.encoding.CompositeEncoder
import kotlinx.serialization.modules.EmptySerializersModule
import kotlinx.serialization.modules.SerializersModule
import okio.Buffer
import okio.BufferedSink
import okio.BufferedSource
val kxsBinary: KxsBinary = KxsBinary()
@ -27,18 +24,18 @@ class KxsBinary(
) : BinaryFormat {
override fun <T> encodeToByteArray(serializer: SerializationStrategy<T>, value: T): ByteArray {
val output = ByteArrayOutputStream()
val encoder = KxsDataOutputEncoder(DataOutputStream(output), serializersModule)
val output = Buffer()
val encoder = KxsDataOutputEncoder(output, serializersModule)
encoder.encodeSerializableValue(serializer, value)
return output.toByteArray()
return output.readByteArray()
}
override fun <T> decodeFromByteArray(
deserializer: DeserializationStrategy<T>,
bytes: ByteArray
): T {
val input = ByteArrayInputStream(bytes)
val decoder = KxsDataInputDecoder(DataInputStream(input), serializersModule)
val input = Buffer().write(bytes)
val decoder = KxsDataInputDecoder(input, serializersModule)
return decoder.decodeSerializableValue(deserializer)
}
}
@ -46,28 +43,31 @@ class KxsBinary(
private val byteArraySerializer = ByteArraySerializer()
class KxsDataOutputEncoder(
private val output: DataOutput,
private val output: BufferedSink,
override val serializersModule: SerializersModule = EmptySerializersModule,
) : AbstractEncoder() {
override fun encodeBoolean(value: Boolean) = output.writeByte(if (value) 1 else 0)
override fun encodeByte(value: Byte) = output.writeByte(value.toInt())
override fun encodeChar(value: Char) = output.writeChar(value.code)
override fun encodeDouble(value: Double) = output.writeDouble(value)
override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output.writeInt(index)
override fun encodeFloat(value: Float) = output.writeFloat(value)
override fun encodeInt(value: Int) = output.writeInt(value)
override fun encodeLong(value: Long) = output.writeLong(value)
override fun encodeShort(value: Short) = output.writeShort(value.toInt())
override fun encodeString(value: String) = output.writeUTF(value)
// override fun encodeString(value: String) = encodeByteArray(value.toByteArray(Charsets.UTF_8))
private operator fun BufferedSink.invoke(action: BufferedSink.() -> Unit): Unit = action()
override fun encodeBoolean(value: Boolean) = output { writeByte(if (value) 1 else 0) }
override fun encodeByte(value: Byte) = output { writeByte(value.toInt()) }
override fun encodeChar(value: Char) = output { writeUtf8CodePoint(value.code) }
override fun encodeDouble(value: Double) = output { writeLong(value.toRawBits()) }
override fun encodeEnum(enumDescriptor: SerialDescriptor, index: Int) = output { writeInt(index) }
override fun encodeFloat(value: Float) = output { writeInt(value.toRawBits()) }
override fun encodeInt(value: Int) = output { writeInt(value) }
override fun encodeLong(value: Long) = output { writeLong(value) }
override fun encodeShort(value: Short) = output { writeShort(value.toInt()) }
override fun encodeString(value: String) = output {
encodeCompactSize(value.length)
writeUtf8(value)
}
override fun beginCollection(
descriptor: SerialDescriptor,
collectionSize: Int
): CompositeEncoder {
encodeInt(collectionSize)
// encodeCompactSize(collectionSize)
encodeCompactSize(collectionSize)
return this
}
@ -98,7 +98,7 @@ class KxsDataOutputEncoder(
class KxsDataInputDecoder(
private val input: DataInput,
private val input: BufferedSource,
override val serializersModule: SerializersModule = EmptySerializersModule,
private var elementsCount: Int = 0,
) : AbstractDecoder() {
@ -107,15 +107,17 @@ class KxsDataInputDecoder(
override fun decodeBoolean(): Boolean = input.readByte().toInt() != 0
override fun decodeByte(): Byte = input.readByte()
override fun decodeChar(): Char = input.readChar()
override fun decodeDouble(): Double = input.readDouble()
override fun decodeChar(): Char = input.readUtf8CodePoint().toChar()
override fun decodeDouble(): Double = Double.fromBits(input.readLong())
override fun decodeEnum(enumDescriptor: SerialDescriptor): Int = input.readInt()
override fun decodeFloat(): Float = input.readFloat()
override fun decodeFloat(): Float = Float.fromBits(input.readInt())
override fun decodeInt(): Int = input.readInt()
override fun decodeLong(): Long = input.readLong()
override fun decodeShort(): Short = input.readShort()
override fun decodeString(): String = input.readUTF()
// override fun decodeString(): String = decodeByteArray().toString(Charsets.UTF_8)
override fun decodeString(): String {
val size = decodeCompactSize()
return input.readUtf8(size.toLong())
}
override fun decodeElementIndex(descriptor: SerialDescriptor): Int {
if (elementIndex == elementsCount) return CompositeDecoder.DECODE_DONE
@ -128,8 +130,7 @@ class KxsDataInputDecoder(
override fun decodeSequentially(): Boolean = true
override fun decodeCollectionSize(descriptor: SerialDescriptor): Int =
decodeInt().also { elementsCount = it }
// decodeCompactSize().also { elementsCount = it }
decodeCompactSize().also { elementsCount = it }
override fun decodeNotNullMark(): Boolean = decodeBoolean()

22
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/serdes/kxsJson.kt
View file

@ -1,7 +1,7 @@
package dev.adamko.kafkatorio.processor.serdes
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
import dev.adamko.kafkatorio.events.schema.jsonMapperKafkatorio
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
@ -13,15 +13,15 @@ import org.http4k.format.asConfigurable
import org.http4k.format.withStandardMappings
object KafkatorioPacketSerde : Serde<KafkatorioPacket> {
override fun serializer() = Serializer<KafkatorioPacket> { _, message ->
jsonMapper.encodeToString<KafkatorioPacket>(message).encodeToByteArray()
}
override fun deserializer() = Deserializer<KafkatorioPacket> { _, bytes ->
jsonMapper.decodeFromString(bytes.decodeToString())
}
}
//object KafkatorioPacketSerde : Serde<KafkatorioPacket2> {
// override fun serializer() = Serializer<KafkatorioPacket> { _, message ->
// jsonMapper.encodeToString<KafkatorioPacket>(message).encodeToByteArray()
// }
//
// override fun deserializer() = Deserializer<KafkatorioPacket> { _, bytes ->
// jsonMapper.decodeFromString(bytes.decodeToString())
// }
//}
val jsonMapper: Json = Json {
prettyPrint = true

36
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt
View file

@ -1,13 +1,10 @@
package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.events.schema.FactorioConfigurationUpdate
import dev.adamko.kafkatorio.events.schema.FactorioEvent
import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
import dev.adamko.kafkatorio.processor.KafkatorioTopology
import dev.adamko.kafkatorio.processor.admin.KafkatorioKafkaAdmin
import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.processor.admin.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.streams.map
@ -22,25 +19,24 @@ import org.apache.kafka.streams.processor.RecordContext
fun factorioServerPacketStream(
builder: StreamsBuilder = StreamsBuilder(),
): KStream<FactorioServerId, KafkatorioPacket> {
): KStream<FactorioServerId, KafkatorioPacket2> {
return builder.stream(
KafkatorioTopology.TOPIC_SRC_SERVER_LOG,
TOPIC_SRC_SERVER_LOG,
consumedAs(
"read-raw-packets-from-server",
Serdes.String(),
Serdes.String(),
)
)
.map("decode-packets") { serverId: String, value: String ->
val packet = jsonMapper.decodeFromString<KafkatorioPacket>(value)
).map("decode-packets") { serverId: String, value: String ->
val packet = jsonMapper.decodeFromString<KafkatorioPacket2>(value)
FactorioServerId(serverId) to packet
}
}
fun splitFactorioServerPacketStream(
factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket>
factorioServerPacketStream: KStream<FactorioServerId, KafkatorioPacket2>
) {
factorioServerPacketStream
.to(
@ -49,18 +45,8 @@ fun splitFactorioServerPacketStream(
jsonMapper.serde(),
jsonMapper.serde(),
)
) { _: FactorioServerId, value: KafkatorioPacket, _: RecordContext ->
) { _: FactorioServerId, value: KafkatorioPacket2, _: RecordContext ->
// println("[$key] sending event:${value.eventType} to topic:${value.data.objectName()}")
val base = KafkatorioKafkaAdmin.topicNameBase(value.packetType)
val type: String = when (value) {
is FactorioEvent -> value.data.objectName.name
is FactorioConfigurationUpdate -> "FactorioConfigurationUpdate"
is FactorioPrototypes -> "all"
is FactorioEventUpdatePacket -> value.update.updateType.name
}
"$base.$type"
value.data.topicName
}
}

62
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/flowMapTiles.kt
View file

@ -4,12 +4,8 @@ import com.sksamuel.scrimage.ImmutableImage
import com.sksamuel.scrimage.ScaleMethod
import com.sksamuel.scrimage.color.RGBColor
import com.sksamuel.scrimage.nio.PngWriter
import dev.adamko.kafkatorio.events.schema.ColourHex
import dev.adamko.kafkatorio.events.schema.MapChunkPosition
import dev.adamko.kafkatorio.events.schema.MapTilePosition
import dev.adamko.kafkatorio.events.schema.SurfaceIndex
import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
import dev.adamko.kafkatorio.processor.KafkatorioTopology
import dev.adamko.kafkatorio.schema.common.*
import dev.adamko.kafkatorio.processor.admin.TOPIC_GROUPED_MAP_CHUNKS_STATE
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.streams.forEach
@ -17,7 +13,9 @@ import dev.adamko.kotka.kxs.serde
import java.awt.image.BufferedImage
import java.io.File
import kotlin.coroutines.CoroutineContext
import kotlin.io.path.fileSize
import kotlin.math.abs
import kotlin.time.Duration.Companion.seconds
import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@ -25,17 +23,18 @@ import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.debounce
import kotlinx.coroutines.flow.distinctUntilChangedBy
import kotlinx.coroutines.flow.emitAll
import kotlinx.coroutines.flow.flatMapConcat
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.runningFold
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.supervisorScope
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology
@ -44,10 +43,6 @@ import org.apache.kafka.streams.kstream.KStream
private const val WEB_MAP_TILE_SIZE_PX = 256
private val cScope = CoroutineScope(
Dispatchers.Default + SupervisorJob() + CoroutineName("f-server-map-updates")
)
/** A bridge between Kafka and Kotlin */
private val serverMapChunkHandler = ServerMapChunkHandler()
@ -59,19 +54,17 @@ fun saveMapTiles(
val groupedMapChunkTiles: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
builder.stream(
KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS,
TOPIC_GROUPED_MAP_CHUNKS_STATE,
consumedAs("consume.grouped-map-chunks", kxsBinary.serde(), kxsBinary.serde())
)
groupedMapChunkTiles
.forEach("save-chunked-tiles") { _, value ->
if (value != null) {
runBlocking(Dispatchers.Default) {
println("emitting chunkedTiles ${value.chunkId} tiles: ${value.map.size}")
serverMapChunkHandler.emit(value)
}
}
}
return builder.build()
}
@ -107,9 +100,8 @@ private fun ColourHex.toRgbColor(): RGBColor {
private class ServerMapChunkHandler : CoroutineScope {
override val coroutineContext: CoroutineContext = Dispatchers.Default +
SupervisorJob(cScope.coroutineContext.job) +
CoroutineName("ServerMapChunkHandler")
override val coroutineContext: CoroutineContext =
Dispatchers.Default + SupervisorJob() + CoroutineName("ServerMapChunkHandler")
/** The inbox for this handler. */
@ -119,11 +111,9 @@ private class ServerMapChunkHandler : CoroutineScope {
onBufferOverflow = BufferOverflow.SUSPEND,
)
fun emit(mapChunkTiles: ServerMapChunkTiles<ColourHex>) {
launch {
fun emit(mapChunkTiles: ServerMapChunkTiles<ColourHex>) = launch {
allChunksFlow.emit(mapChunkTiles)
}
}
/** This flow saves images one-by-one */
private val saveImagesFlow = MutableSharedFlow<Cmd.SaveImage>(
@ -135,17 +125,18 @@ private class ServerMapChunkHandler : CoroutineScope {
init {
allChunksFlow
.runningFold(mapOf<ServerMapChunkId, MutableSharedFlow<Cmd.ChunkSubdivide>>()) { acc, src ->
.runningFold(mutableMapOf<ServerMapChunkId, MutableSharedFlow<Cmd.ChunkSubdivide>>()) { acc, src ->
val flow = acc.getOrElse(src.chunkId) {
val flow = acc.getOrPut(src.chunkId) {
println("creating new subdivide-flow for chunkId ${src.chunkId}")
createSubdivisionFlow()
}
flow.emit(Cmd.ChunkSubdivide(src.chunkId, src.map))
acc + (src.chunkId to flow)
acc
}
.distinctUntilChangedBy { it.size }
.onEach {
println("allChunksFlow has ${it.size} subdivide-flows: ${it.keys.joinToString()}")
}
@ -160,9 +151,15 @@ private class ServerMapChunkHandler : CoroutineScope {
println("created new map tile parentFile directory ${chunkImageFile.absolutePath}")
}
val sizeBefore = chunkImageFile.takeIf { it.exists() }?.toPath()?.fileSize()
val savedTile = img.output(PngWriter.NoCompression, chunkImageFile)
if (filename.value.contains("z-1")) {
println("savedTile: $savedTile")
val sizeAfter = savedTile.takeIf { it.exists() }?.toPath()?.fileSize()
if (sizeBefore != sizeAfter) {
// println("savedTile $sizeBefore/$sizeAfter: $savedTile")
} else {
println("savedTile NO-SIZE-CHANGE $sizeBefore/$sizeAfter: $savedTile")
}
}
.launchIn(this)
@ -183,17 +180,20 @@ private class ServerMapChunkHandler : CoroutineScope {
onBufferOverflow = BufferOverflow.SUSPEND,
)
val saveImgCommands = subdivisionFlow
val saveImgCommands: Flow<Cmd.SaveImage> =
subdivisionFlow
.debounce(30.seconds)
.flatMapConcat { subdivideCmd ->
ChunkSize
.values
.entries
.flatMap { chunkSize ->
subdivideCmd.tiles
subdivideCmd
.tiles
.entries
.groupBy(
{ (tile, _) -> tile.toMapChunkPosition(chunkSize.tilesPerChunk) }
) {
it.key to it.value
) { (tilePosition, colour) ->
tilePosition to colour
}
.map { (chunkPos, tiles) ->
async {

234
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt
View file

@ -1,52 +1,31 @@
package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.events.schema.ColourHex
import dev.adamko.kafkatorio.events.schema.FactorioEventUpdate
import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
import dev.adamko.kafkatorio.events.schema.MapChunkPosition
import dev.adamko.kafkatorio.events.schema.MapChunkUpdate
import dev.adamko.kafkatorio.events.schema.MapTile
import dev.adamko.kafkatorio.events.schema.MapTileDictionary.Companion.isNullOrEmpty
import dev.adamko.kafkatorio.events.schema.MapTilePosition
import dev.adamko.kafkatorio.events.schema.MapTiles
import dev.adamko.kafkatorio.events.schema.SurfaceIndex
import dev.adamko.kafkatorio.events.schema.converters.toMapChunkPosition
import dev.adamko.kafkatorio.processor.KafkatorioTopology
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.processor.admin.TOPIC_GROUPED_MAP_CHUNKS_STATE
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapTile
import dev.adamko.kafkatorio.schema.common.MapTileDictionary.Companion.isNullOrEmpty
import dev.adamko.kafkatorio.schema.common.MapTilePosition
import dev.adamko.kafkatorio.schema.common.MapTiles
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.toMapChunkPosition
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kotka.extensions.groupedAs
import dev.adamko.kotka.extensions.materializedWith
import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.repartitionedAs
import dev.adamko.kotka.extensions.streams.filter
import dev.adamko.kotka.extensions.streams.flatMap
import dev.adamko.kotka.extensions.streams.mapValues
import dev.adamko.kotka.extensions.streams.peek
import dev.adamko.kotka.extensions.streams.reduce
import dev.adamko.kotka.extensions.streams.toTable
import dev.adamko.kotka.extensions.tableJoined
import dev.adamko.kotka.extensions.tables.filter
import dev.adamko.kotka.extensions.tables.join
import dev.adamko.kotka.extensions.tables.mapValues
import dev.adamko.kotka.extensions.tables.toStream
import dev.adamko.kotka.kxs.serde
import kotlin.collections.List
import kotlin.collections.Map
import kotlin.collections.associate
import kotlin.collections.component1
import kotlin.collections.component2
import kotlin.collections.getOrElse
import kotlin.collections.groupBy
import kotlin.collections.isNotEmpty
import kotlin.collections.isNullOrEmpty
import kotlin.collections.joinToString
import kotlin.collections.map
import kotlin.collections.mapValues
import kotlin.collections.mutableSetOf
import kotlin.collections.plus
import kotlinx.serialization.Serializable
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology
@ -72,33 +51,39 @@ data class ServerMapChunkTiles<Data>(
}
fun groupMapChunks(
builder: StreamsBuilder,
prototypesTopicName: String = "kafkatorio.${KafkatorioPacket.PacketType.PROTOTYPES}.all",
mapChunkUpdatesTopicName: String = "kafkatorio.${KafkatorioPacket.PacketType.UPDATE}.${FactorioEventUpdate.FactorioEventUpdateType.MAP_CHUNK}",
): Topology {
fun groupMapChunks(builder: StreamsBuilder): Topology {
val mapChunksStream: KStream<FactorioServerId, MapChunkUpdate> = builder.streamPacketData()
val protosStream: KStream<FactorioServerId, PrototypesUpdate> = builder.streamPacketData()
// val protosStream: KStream<FactorioServerId, KafkatorioPacket2> =
// builder.stream<FactorioServerId?, KafkatorioPacket2?>(
// prototypesTopicName,
// consumedAs("consume.factorio-protos.all", jsonMapper.serde(), jsonMapper.serde())
// ).filter { key: FactorioServerId?, value: KafkatorioPacket2? ->
// key != null
// && value != null
// && value.data ==
// }
// .map
val protosStream: KStream<FactorioServerId, FactorioPrototypes> =
builder.stream(
prototypesTopicName,
consumedAs("consume.factorio-protos.all", jsonMapper.serde(), jsonMapper.serde())
)
val tileProtoColourDict: KTable<FactorioServerId, TileColourDict> =
tileProtoColourDictionary(protosStream)
val mapChunksUpdatesStream: KStream<FactorioServerId, FactorioEventUpdatePacket> =
builder.stream(
mapChunkUpdatesTopicName,
consumedAs(
"consume.map-chunk-updates",
jsonMapper.serde<FactorioServerId>(),
jsonMapper.serde<FactorioEventUpdatePacket>(),
)
)
// val mapChunksUpdatesStream: KStream<FactorioServerId, FactorioEventUpdatePacket> =
// builder.stream(
// mapChunkUpdatesTopicName,
// consumedAs(
// "consume.map-chunk-updates",
// jsonMapper.serde<FactorioServerId>(),
// jsonMapper.serde<FactorioEventUpdatePacket>(),
// )
// )
val groupedMapChunkTiles: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
groupTilesIntoChunksWithColours(
mapChunksUpdatesStream,
mapChunksStream,
tileProtoColourDict,
)
@ -108,13 +93,13 @@ fun groupMapChunks(
!v?.map.isNullOrEmpty()
}
.mapValues("stream-grouped-map-tiles.map-not-null") { _, v ->
v as ServerMapChunkTiles<ColourHex>
requireNotNull(v)
}
.peek("stream-grouped-map-tiles.peek") { _, v ->
println("Grouping map tiles result: ${v.chunkId} / ${v.map.size}")
}
.to(
KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS,
TOPIC_GROUPED_MAP_CHUNKS_STATE,
producedAs(
"produce.grouped-map-chunks",
kxsBinary.serde<ServerMapChunkId>(),
@ -127,37 +112,24 @@ fun groupMapChunks(
private fun groupTilesIntoChunksWithColours(
mapChunkUpdatePacketsStream: KStream<FactorioServerId, FactorioEventUpdatePacket>,
mapChunksStream: KStream<FactorioServerId, MapChunkUpdate>,
tileProtoColourDict: KTable<FactorioServerId, TileColourDict>,
): KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> {
val mapChunkUpdatesStream: KStream<FactorioServerId, MapTiles> =
mapChunkUpdatePacketsStream
.filter { _, packet: FactorioEventUpdatePacket? ->
when (val data = packet?.update) {
is MapChunkUpdate -> !data.tileDictionary.isNullOrEmpty()
else -> false
}
}
.mapValues("map-chunk-update-packets.convert-to-map-tiles") { _, packet: FactorioEventUpdatePacket ->
val mapChunkUpdate: MapChunkUpdate = requireNotNull(packet.update as? MapChunkUpdate) {
"invalid UpdatePacket type $packet"
}
val tiles: List<MapTile> = requireNotNull(mapChunkUpdate.tileDictionary?.toMapTileList()) {
"invalid MapChunkUpdate packet, tiles must not be null"
}
require(tiles.isNotEmpty()) {
"invalid MapChunkUpdate packet, tiles must not be empty"
}
MapTiles(mapChunkUpdate.surfaceIndex, tiles)
val mapTilesStream: KStream<FactorioServerId, MapTiles> =
mapChunksStream.filter { _, value ->
!value.tileDictionary.isNullOrEmpty()
}.mapValues("map-chunk-update-packets.convert-to-map-tiles") { _, packet: MapChunkUpdate ->
val mapTiles = packet.tileDictionary?.toMapTileList() ?: emptyList()
MapTiles(packet.key.surfaceIndex, mapTiles)
}.filter { _, value ->
value.tiles.isNotEmpty()
}
val chunkedTilesTable: KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>> =
mapChunkUpdatesStream
mapTilesStream
// group tiles by server & surface & chunk
.flatMap("server-map-data.tiles.flatMapByChunk") { serverId: FactorioServerId, mapTiles: MapTiles ->
@ -208,65 +180,65 @@ private fun groupTilesIntoChunksWithColours(
kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
)
) { chunkTiles, other -> chunkTiles + other }
.toStream("server-map-data.tiles.repartition.to-stream")
.repartition(
repartitionedAs(
"server-map-data.tiles.repartition.config",
kxsBinary.serde<ServerMapChunkId>(),
kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
3,
)
).toTable(
"server-map-data.tiles.repartition.to-table",
materializedWith(
kxsBinary.serde<ServerMapChunkId>(),
kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
)
)
.filter("server-map-data.tiles.repartition.filter-not-null") { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
v != null
}
.mapValues(
"server-map-data.tiles.repartition.map-not-null",
materializedWith(
kxsBinary.serde<ServerMapChunkId>(),
kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
)
) { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
v as ServerMapChunkTiles<TileProtoHashCode>
}
// .toStream("server-map-data.tiles.repartition.to-stream")
// .repartition(
// repartitionedAs(
// "server-map-data.tiles.repartition.config",
// kxsBinary.serde<ServerMapChunkId>(),
// kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
// 3,
// )
// ).toTable(
// "server-map-data.tiles.repartition.to-table",
// materializedWith(
// kxsBinary.serde<ServerMapChunkId>(),
// kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
// )
// )
// .filter("server-map-data.tiles.repartition.filter-not-null") { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
// v != null
// }
// .mapValues(
// "server-map-data.tiles.repartition.map-not-null",
// materializedWith(
// kxsBinary.serde<ServerMapChunkId>(),
// kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
// )
// ) { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
// v as ServerMapChunkTiles<TileProtoHashCode>
// }
// repartition to try and fix KTable-KTable joining skipping records
val tileProtoColourDictRepartitioned =
tileProtoColourDict
.toStream("tileProtoColourDict.repartition.to-stream")
.repartition(
repartitionedAs(
"tileProtoColourDict.repartition.config",
kxsBinary.serde<FactorioServerId>(),
kxsBinary.serde<TileColourDict?>(),
3,
)
).toTable(
"tileProtoColourDict.repartition.to-table",
materializedWith(
kxsBinary.serde<FactorioServerId>(),
kxsBinary.serde<TileColourDict?>(),
)
)
.filter("tileProtoColourDict.repartition.filter-not-null") { _: FactorioServerId, v: TileColourDict? ->
v != null
}
.mapValues(
"tileProtoColourDict.repartition.map-not-null",
materializedWith(
kxsBinary.serde<FactorioServerId>(),
kxsBinary.serde<TileColourDict>(),
)
) { _: FactorioServerId, v: TileColourDict? ->
v as TileColourDict
}
// .toStream("tileProtoColourDict.repartition.to-stream")
// .repartition(
// repartitionedAs(
// "tileProtoColourDict.repartition.config",
// kxsBinary.serde<FactorioServerId>(),
// kxsBinary.serde<TileColourDict?>(),
// 3,
// )
// ).toTable(
// "tileProtoColourDict.repartition.to-table",
// materializedWith(
// kxsBinary.serde<FactorioServerId>(),
// kxsBinary.serde<TileColourDict?>(),
// )
// )
// .filter("tileProtoColourDict.repartition.filter-not-null") { _: FactorioServerId, v: TileColourDict? ->
// v != null
// }
// .mapValues(
// "tileProtoColourDict.repartition.map-not-null",
// materializedWith(
// kxsBinary.serde<FactorioServerId>(),
// kxsBinary.serde<TileColourDict>(),
// )
// ) { _: FactorioServerId, v: TileColourDict? ->
// v as TileColourDict
// }
val colourisedChunkTable: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =

8
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/model.kt
View file

@ -36,11 +36,11 @@ enum class ChunkSize(
companion object {
// cache values for better performance. KT-48872
val values: List<ChunkSize> = values().asList()
val entries: Set<ChunkSize> = values().toSet()
val MAX: ChunkSize = values.maxByOrNull { it.tilesPerChunk }!!
val MIN: ChunkSize = values.minByOrNull { it.tilesPerChunk }!!
val STANDARD: ChunkSize = values.first { it.zoomLevel == 0 }
val MAX: ChunkSize = entries.maxByOrNull { it.tilesPerChunk }!!
val MIN: ChunkSize = entries.minByOrNull { it.tilesPerChunk }!!
val STANDARD: ChunkSize = entries.first { it.zoomLevel == 0 }
}
}

27
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/packetDataStreams.kt Normal file
View file

@ -0,0 +1,27 @@
package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.processor.admin.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema2.KafkatorioPacketData2
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.extensions.streams.mapValues
import dev.adamko.kotka.kxs.serde
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.KStream
inline fun <reified T : KafkatorioPacketData2> StreamsBuilder.streamPacketData(): KStream<FactorioServerId, T> {
val simpleName = T::class.simpleName!!
val topicName = T::class.topicName
return stream<FactorioServerId, KafkatorioPacket2>(
topicName,
consumedAs("consume.${simpleName}", jsonMapper.serde(), jsonMapper.serde())
).filter { key: FactorioServerId?, value: KafkatorioPacket2? ->
key != null && value?.data is T
}.mapValues("consume.map-values.${simpleName}") { _: FactorioServerId, packet: KafkatorioPacket2 ->
packet.data as T
}
}

12
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/playerUpdates.kt
View file

@ -1,9 +1,10 @@
package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.events.schema.FactorioEventUpdate
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
import dev.adamko.kafkatorio.processor.WebsocketServer
import dev.adamko.kafkatorio.processor.admin.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema2.PlayerUpdate
import dev.adamko.kotka.extensions.consumedAs
import dev.adamko.kotka.kxs.serde
import kotlinx.serialization.encodeToString
@ -14,11 +15,10 @@ fun playerUpdatesToWsServer(
websocketServer: WebsocketServer,
builder: StreamsBuilder = StreamsBuilder(),
) {
builder.stream<FactorioServerId, KafkatorioPacket>(
"kafkatorio.${KafkatorioPacket.PacketType.UPDATE}.${FactorioEventUpdate.FactorioEventUpdateType.PLAYER}",
builder.stream<FactorioServerId, KafkatorioPacket2>(
PlayerUpdate::class.topicName,
consumedAs("EventUpdate.Player-updates-for-ws-server", jsonMapper.serde(), jsonMapper.serde()),
)
.foreach { _, value ->
).foreach { _, value ->
// println("sending ${value.packetType} packet to websocket")
websocketServer.sendMessage(jsonMapper.encodeToString(value))
}

60
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt
View file

@ -1,15 +1,13 @@
package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.events.schema.ColourHex
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
import dev.adamko.kafkatorio.events.schema.MapTile
import dev.adamko.kafkatorio.events.schema.MapTilePrototype
import dev.adamko.kafkatorio.events.schema.converters.toHex
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.MapTile
import dev.adamko.kafkatorio.schema.common.toHex
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
import dev.adamko.kafkatorio.schema.prototypes.MapTilePrototype
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kotka.extensions.materializedAs
import dev.adamko.kotka.extensions.materializedWith
import dev.adamko.kotka.extensions.streams.filter
import dev.adamko.kotka.extensions.streams.mapValues
import dev.adamko.kotka.extensions.streams.toTable
@ -25,6 +23,7 @@ import org.apache.kafka.streams.kstream.KTable
value class TileProtoHashCode private constructor(val code: Int) {
constructor(prototype: MapTilePrototype) : this(prototype.name.hashCode())
constructor(tile: MapTile) : this(tile.proto.hashCode())
constructor(tile: FactorioPrototype2.MapTile) : this(tile.name.hashCode())
}
@ -35,26 +34,21 @@ value class TileColourDict(val map: Map<TileProtoHashCode, ColourHex>)
/** Get the latest map-tile colours per server. */
fun tileProtoColourDictionary(
factorioServerPacketStream: KStream<FactorioServerId, FactorioPrototypes>
factorioServerPacketStream: KStream<FactorioServerId, PrototypesUpdate>
): KTable<FactorioServerId, TileColourDict> {
// get all the prototypes
return factorioServerPacketStream
.mapValues("server-map-data.tile-prototypes.mapValues") { _, packet: KafkatorioPacket ->
val map = when (packet) {
is FactorioPrototypes -> packet
return factorioServerPacketStream.mapValues("server-map-data.tile-prototypes.map-values") { _, protoPacket: PrototypesUpdate ->
val map = protoPacket
.prototypes
.filterIsInstance<MapTilePrototype>()
.filterIsInstance<FactorioPrototype2.MapTile>()
.associate { TileProtoHashCode(it) to it.mapColour.toHex() }
else -> mapOf()
}
TileColourDict(map)
}.filter("server-map-data.tile-prototypes.filterMapTileProtos") { _, dict: TileColourDict ->
dict.map.isNotEmpty()
}.peek { serverId, dict: TileColourDict ->
println("server $serverId has TileColourDict[${dict.map.size}]: ${dict.map.entries.joinToString()}")
}
.toTable(
}.toTable(
"server-map-data.tile-prototypes",
materializedAs(
"server-map-data.tile-prototypes.store",
@ -62,5 +56,31 @@ fun tileProtoColourDictionary(
kxsBinary.serde(),
)
)
}
// get all the prototypes
// return factorioServerPacketStream
// .mapValues("server-map-data.tile-prototypes.mapValues") { _, packet: KafkatorioPacket ->
// val map = when (packet) {
// is FactorioPrototype -> packet
// .prototypes
// .filterIsInstance<MapTilePrototype>()
// .associate { TileProtoHashCode(it) to it.mapColour.toHex() }
// else -> mapOf()
// }
// TileColourDict(map)
// }.filter("server-map-data.tile-prototypes.filterMapTileProtos") { _, dict: TileColourDict ->
// dict.map.isNotEmpty()
// }.peek { serverId, dict: TileColourDict ->
// println("server $serverId has TileColourDict[${dict.map.size}]: ${dict.map.entries.joinToString()}")
// }
// .toTable(
// "server-map-data.tile-prototypes",
// materializedAs(
// "server-map-data.tile-prototypes.store",
// kxsBinary.serde(),
// kxsBinary.serde(),
// )
// )
//
//}

252
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/KxsBinaryTests.kt
View file

@ -1,24 +1,59 @@
package dev.adamko.kafkatorio.processor.misc
import dev.adamko.kafkatorio.events.schema.MapChunkPosition
import dev.adamko.kafkatorio.events.schema.SurfaceIndex
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kafkatorio.processor.topology.ChunkSize
import dev.adamko.kafkatorio.processor.topology.FactorioServerId
import dev.adamko.kafkatorio.processor.topology.ServerMapChunkId
import dev.adamko.kafkatorio.processor.topology.ServerMapChunkTiles
import dev.adamko.kafkatorio.schema.common.ColourHex
import dev.adamko.kafkatorio.schema.common.MapChunkPosition
import dev.adamko.kafkatorio.schema.common.MapTilePosition
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.collections.shouldContainExactly
import io.kotest.matchers.comparables.shouldBeEqualComparingTo
import io.kotest.matchers.equality.shouldBeEqualToComparingFields
import io.kotest.matchers.ints.shouldBeExactly
import io.kotest.matchers.longs.shouldBeExactly
import io.kotest.matchers.shouldBe
import io.kotest.property.Arb
import io.kotest.property.arbitrary.boolean
import io.kotest.property.arbitrary.booleanArray
import io.kotest.property.arbitrary.byte
import io.kotest.property.arbitrary.byteArray
import io.kotest.property.arbitrary.char
import io.kotest.property.arbitrary.charArray
import io.kotest.property.arbitrary.double
import io.kotest.property.arbitrary.doubleArray
import io.kotest.property.arbitrary.float
import io.kotest.property.arbitrary.floatArray
import io.kotest.property.arbitrary.int
import io.kotest.property.arbitrary.intArray
import io.kotest.property.arbitrary.long
import io.kotest.property.arbitrary.longArray
import io.kotest.property.arbitrary.positiveInt
import io.kotest.property.arbitrary.short
import io.kotest.property.arbitrary.shortArray
import io.kotest.property.arbitrary.string
import io.kotest.property.arbitrary.uByte
import io.kotest.property.arbitrary.uInt
import io.kotest.property.arbitrary.uLong
import io.kotest.property.arbitrary.uShort
import io.kotest.property.checkAll
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToByteArray
import kotlinx.serialization.encodeToString
class KxsBinaryTests : FunSpec({
test("byte array") {
val ba = byteArrayOf(
0, 4, 116, 101, 115, 116, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
4, 116, 101, 115, 116, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
)
val result = kxsBinary.decodeFromByteArray(ServerMapChunkId.serializer(), ba)
@ -47,7 +82,7 @@ class KxsBinaryTests : FunSpec({
withClue(ba.toString(Charsets.UTF_8)) {
ba shouldBe byteArrayOf(
0, 4, 116, 101, 115, 116, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
4, 116, 101, 115, 116, 2, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 3
)
}
}
@ -55,7 +90,10 @@ class KxsBinaryTests : FunSpec({
context("inline value class") {
val uuid = "fa858d73-6205-44cf-aeeb-6bbf3148c4a7"
//@formatter:off
val expectedBytes = byteArrayOf(0, 36, 102, 97, 56, 53, 56, 100, 55, 51, 45, 54, 50, 48, 53, 45, 52, 52, 99, 102, 45, 97, 101, 101, 98, 45, 54, 98, 98, 102, 51, 49, 52, 56, 99, 52, 97, 55)
val expectedBytes = byteArrayOf(
36, 102, 97, 56, 53, 56, 100, 55, 51, 45, 54, 50, 48, 53, 45, 52, 52, 99,
102, 45, 97, 101, 101, 98, 45, 54, 98, 98, 102, 51, 49, 52, 56, 99, 52, 97, 55,
)
//@formatter:on
test("encode") {
@ -73,4 +111,208 @@ class KxsBinaryTests : FunSpec({
}
}
context("ServerMapChunkTiles") {
val initial = ServerMapChunkTiles(
ServerMapChunkId(
FactorioServerId("server-id"),
MapChunkPosition(33, 44),
SurfaceIndex(4u),
ChunkSize.CHUNK_032,
),
mapOf(
MapTilePosition(1, 2) to ColourHex(1u, 2u, 3u, 4u),
)
)
test("!json round trip") {
val jsonEncoded = jsonMapperKafkatorio.encodeToString(initial)
val jsonDecoded =
jsonMapperKafkatorio.decodeFromString<ServerMapChunkTiles<ColourHex>>(jsonEncoded)
jsonDecoded shouldBeEqualToComparingFields initial
}
test("binary round trip") {
val binaryEncoded = kxsBinary.encodeToByteArray(initial)
val binaryDecoded = kxsBinary.decodeFromByteArray<ServerMapChunkTiles<ColourHex>>(binaryEncoded)
binaryDecoded shouldBeEqualToComparingFields initial
}
}
context("primitives round trip") {
test("double") {
checkAll(Arb.double()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Double = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeEqualComparingTo input
}
}
test("float") {
checkAll(Arb.float()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Float = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeEqualComparingTo input
}
}
test("int") {
checkAll(Arb.int()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Int = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeExactly input
}
}
test("long") {
checkAll(Arb.long()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Long = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeExactly input
}
}
test("short") {
checkAll(Arb.short()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Short = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeEqualComparingTo input
}
}
test("byte") {
checkAll(Arb.byte()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Byte = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeEqualComparingTo input
}
}
test("boolean") {
checkAll(Arb.boolean()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Boolean = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeEqualComparingTo input
}
}
test("char") {
checkAll(Arb.char()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: Char = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeEqualComparingTo input
}
}
test("string") {
checkAll(Arb.string()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: String = kxsBinary.decodeFromByteArray(encoded)
decoded shouldBeEqualComparingTo input
}
}
}
@OptIn(ExperimentalUnsignedTypes::class)
context("unsigned numbers round trip") {
test("uint") {
checkAll(Arb.uInt()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(UInt.serializer(), input)
val decoded: UInt = kxsBinary.decodeFromByteArray(UInt.serializer(), encoded)
decoded shouldBeEqualComparingTo input
}
}
test("ulong") {
checkAll(Arb.uLong()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(ULong.serializer(), input)
val decoded: ULong = kxsBinary.decodeFromByteArray(ULong.serializer(), encoded)
decoded shouldBeEqualComparingTo input
}
}
test("ushort") {
checkAll(Arb.uShort()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(UShort.serializer(), input)
val decoded: UShort = kxsBinary.decodeFromByteArray(UShort.serializer(), encoded)
decoded shouldBeEqualComparingTo input
}
}
test("ubyte") {
checkAll(Arb.uByte()) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(UByte.serializer(), input)
val decoded: UByte = kxsBinary.decodeFromByteArray(UByte.serializer(), encoded)
decoded shouldBeEqualComparingTo input
}
}
}
context("primitive arrays round trip") {
test("double array") {
checkAll(Arb.doubleArray(Arb.positiveInt(50), Arb.double())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: DoubleArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
test("float array") {
checkAll(Arb.floatArray(Arb.positiveInt(50), Arb.float())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: FloatArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
test("int array") {
checkAll(Arb.intArray(Arb.positiveInt(50), Arb.int())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: IntArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
test("long array") {
checkAll(Arb.longArray(Arb.positiveInt(50), Arb.long())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: LongArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
test("short array") {
checkAll(Arb.shortArray(Arb.positiveInt(50), Arb.short())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: ShortArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
test("byte array") {
checkAll(Arb.byteArray(Arb.positiveInt(50), Arb.byte())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: ByteArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
test("boolean array") {
checkAll(Arb.booleanArray(Arb.positiveInt(50), Arb.boolean())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: BooleanArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
test("char array") {
checkAll(Arb.charArray(Arb.positiveInt(50), Arb.char())) { input ->
val encoded: ByteArray = kxsBinary.encodeToByteArray(input)
val decoded: CharArray = kxsBinary.decodeFromByteArray(encoded)
decoded.asList() shouldContainExactly input.asList()
}
}
}
})

5
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/misc/serdeSize.kt
View file

@ -1,14 +1,11 @@
package dev.adamko.kafkatorio.processor.misc
import dev.adamko.kafkatorio.events.schema.ColourHex
import dev.adamko.kafkatorio.events.schema.MapChunkPosition
import dev.adamko.kafkatorio.events.schema.MapTilePosition
import dev.adamko.kafkatorio.events.schema.SurfaceIndex
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kafkatorio.processor.topology.ChunkSize
import dev.adamko.kafkatorio.processor.topology.FactorioServerId
import dev.adamko.kafkatorio.processor.topology.ServerMapChunkId
import dev.adamko.kafkatorio.processor.topology.ServerMapChunkTiles
import dev.adamko.kafkatorio.schema.common.*
import java.util.UUID
import kotlinx.serialization.encodeToByteArray
import org.junit.jupiter.api.Test

56
modules/events-server/src/test/kotlin/dev/adamko/kafkatorio/processor/topology/GroupMapChunksTest.kt
View file

@ -1,21 +1,15 @@
package dev.adamko.kafkatorio.processor.topology
import dev.adamko.kafkatorio.events.schema.Colour
import dev.adamko.kafkatorio.events.schema.ColourHex
import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
import dev.adamko.kafkatorio.events.schema.MapChunkPosition
import dev.adamko.kafkatorio.events.schema.MapChunkUpdate
import dev.adamko.kafkatorio.events.schema.MapTileDictionary
import dev.adamko.kafkatorio.events.schema.MapTilePosition
import dev.adamko.kafkatorio.events.schema.MapTilePrototype
import dev.adamko.kafkatorio.events.schema.PrototypeKey
import dev.adamko.kafkatorio.events.schema.PrototypeName
import dev.adamko.kafkatorio.events.schema.SurfaceIndex
import dev.adamko.kafkatorio.events.schema.Tick
import dev.adamko.kafkatorio.processor.KafkatorioTopology
import dev.adamko.kafkatorio.processor.admin.TOPIC_GROUPED_MAP_CHUNKS_STATE
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.processor.serdes.kxsBinary
import dev.adamko.kafkatorio.schema.common.*
import dev.adamko.kafkatorio.schema.common.MapTileDictionary.PrototypeKey
import dev.adamko.kafkatorio.schema.prototypes.FactorioPrototype2
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.MapChunkUpdateKey
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kotka.kxs.serde
import io.kotest.assertions.withClue
import io.kotest.core.spec.style.FunSpec
@ -32,34 +26,29 @@ import org.apache.kafka.streams.TopologyTestDriver
class GroupMapChunksTest : FunSpec({
test("basic test") {
val streamsBuilder = StreamsBuilder()
val prototypesTopicName = "prototypes-topic"
val mapChunkUpdatesTopicName = "map-chunk-updates-topic"
val topology = groupMapChunks(
StreamsBuilder(),
prototypesTopicName,
mapChunkUpdatesTopicName,
)
val topology = groupMapChunks(streamsBuilder)
val testDriver = TopologyTestDriver(topology)
val prototypesTestInputTopic: TestInputTopic<FactorioServerId, FactorioPrototypes> =
val prototypesTestInputTopic: TestInputTopic<FactorioServerId, PrototypesUpdate> =
testDriver.createInputTopic(
prototypesTopicName,
jsonMapper.serde<FactorioServerId>().serializer(),
jsonMapper.serde<FactorioPrototypes>().serializer(),
jsonMapper.serde<PrototypesUpdate>().serializer(),
)
val serverId = FactorioServerId("test-server-id")
prototypesTestInputTopic.pipeInput(
serverId,
FactorioPrototypes(
modVersion = "1.2.3",
tick = Tick(456u),
PrototypesUpdate(
listOf(
MapTilePrototype(
FactorioPrototype2.MapTile(
PrototypeName("fake-proto"),
layer = 1u,
mapColour = Colour(0.3f, 0.2f, 0.1f, 1.0f),
@ -71,26 +60,25 @@ class GroupMapChunksTest : FunSpec({
)
)
val mapChunkUpdatesInputTopic: TestInputTopic<FactorioServerId, FactorioEventUpdatePacket> =
val mapChunkUpdatesInputTopic: TestInputTopic<FactorioServerId, MapChunkUpdate> =
testDriver.createInputTopic(
mapChunkUpdatesTopicName,
jsonMapper.serde<FactorioServerId>().serializer(),
jsonMapper.serde<FactorioEventUpdatePacket>().serializer(),
jsonMapper.serde<MapChunkUpdate>().serializer(),
)
mapChunkUpdatesInputTopic.pipeInput(
serverId,
FactorioEventUpdatePacket(
modVersion = "1.2.3",
tick = Tick(854u),
update = MapChunkUpdate(
MapChunkUpdate(
key = MapChunkUpdateKey(
MapChunkPosition(11, 22),
SurfaceIndex(1u),
),
tileDictionary = MapTileDictionary(
tilesXY = mapOf(
"1" to mapOf("2" to PrototypeKey("33"))
"1" to mapOf("2" to PrototypeKey(33))
),
protos = mapOf(PrototypeKey("33") to PrototypeName("fake-proto"))
)
protos = mapOf(PrototypeName("fake-proto") to PrototypeKey(33))
)
)
)
@ -98,7 +86,7 @@ class GroupMapChunksTest : FunSpec({
val outputTopic: TestOutputTopic<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
testDriver
.createOutputTopic(
KafkatorioTopology.TOPIC_GROUPED_MAP_CHUNKS,
TOPIC_GROUPED_MAP_CHUNKS_STATE,
kxsBinary.serde<ServerMapChunkId>().deserializer(),
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>().deserializer(),
)

4
modules/infra-factorio-server/src/.env
View file

@ -1,4 +1,4 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
FACTORIO_VERSION=1.1.53
KAFKATORIO_VERSION=0.3.2
FACTORIO_VERSION=1.1.57
KAFKATORIO_VERSION=0.4.0

2
modules/infra-kafka-cluster/src/.env
View file

@ -1,3 +1,3 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
KAFKATORIO_VERSION=0.3.2
KAFKATORIO_VERSION=0.4.0

3
modules/infra-kafka-cluster/src/kafka-server.properties
View file

@ -125,3 +125,6 @@ log.retention.check.interval.ms=300000
### record size ###
message.max.bytes=10485760
replica.fetch.max.bytes=10485760
log.cleanup.policy=compact
auto.create.topics.enable=false

2
modules/infra-kafka-pipe/src/.env
View file

@ -1,3 +1,3 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
KAFKATORIO_VERSION=0.3.2
KAFKATORIO_VERSION=0.4.0

2
modules/infra-kafka-pipe/src/docker-compose.yml
View file

@ -2,7 +2,7 @@ version: "3.9"
services:
kafka-pipe:
image: openjdk:11-jre-buster
image: dev-adamko/kafka-pipe/${KAFKATORIO_VERSION}
container_name: "kafka-pipe"
build:
dockerfile: "kafka-pipe.Dockerfile"

2
modules/infra-kafka-pipe/src/kafka-pipe.sh
View file

@ -19,7 +19,7 @@ while :; do
sed -n -e 's/^KafkatorioPacket: //p' |
kcat -P -T \
-b "$KAFKA_HOST" \
-t factorio-server-log \
-t kafkatorio.src.server-log \
-k "$INSTANCE_ID" \
-H "KAFKA-PIPE-VERSION=$KAFKATORIO_VERSION"

4
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/converters.kt
View file

@ -1,7 +1,7 @@
package dev.adamko.kafkatorio.webmap
import dev.adamko.kafkatorio.events.schema.MapEntityPosition
import dev.adamko.kafkatorio.events.schema.PlayerUpdate
import dev.adamko.kafkatorio.schema.common.MapEntityPosition
import dev.adamko.kafkatorio.schema2.PlayerUpdate
import io.kvision.maps.Maps
import io.kvision.maps.externals.leaflet.geo.LatLng

2
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/json.kt
View file

@ -1,6 +1,6 @@
package dev.adamko.kafkatorio.webmap
import dev.adamko.kafkatorio.events.schema.jsonMapperKafkatorio
import dev.adamko.kafkatorio.schema.jsonMapperKafkatorio
import kotlinx.serialization.json.Json
val jsonMapper = Json {

18
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/state.kt
View file

@ -1,12 +1,12 @@
package dev.adamko.kafkatorio.webmap
import dev.adamko.kafkatorio.events.schema.Colour
import dev.adamko.kafkatorio.events.schema.MapEntityPosition
import dev.adamko.kafkatorio.events.schema.PlayerIndex
import dev.adamko.kafkatorio.events.schema.PlayerUpdate
import dev.adamko.kafkatorio.events.schema.SurfaceIndex
import dev.adamko.kafkatorio.events.schema.Tick
import dev.adamko.kafkatorio.events.schema.converters.toHexString
import dev.adamko.kafkatorio.schema.common.Colour
import dev.adamko.kafkatorio.schema.common.MapEntityPosition
import dev.adamko.kafkatorio.schema.common.PlayerIndex
import dev.adamko.kafkatorio.schema.common.SurfaceIndex
import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.common.toHexString
import dev.adamko.kafkatorio.schema2.PlayerUpdate
import io.kvision.maps.Maps
import io.kvision.maps.Maps.Companion.L
import io.kvision.maps.externals.leaflet.geo.LatLng
@ -43,8 +43,8 @@ fun factorioGameReducer(state: FactorioGameState, update: FactorioUpdate): Facto
when (update) {
is FactorioUpdate.Player -> {
val playerState = state.players.getOrElse(update.data.index) {
PlayerState(update.data.index, update.tick)
val playerState = state.players.getOrElse(update.data.key.index) {
PlayerState(update.data.key.index, update.tick)
}
playerState.update(update.tick, update.data)

40
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt
View file

@ -1,13 +1,14 @@
package dev.adamko.kafkatorio.webmap
import dev.adamko.kafkatorio.events.schema.EntityUpdate
import dev.adamko.kafkatorio.events.schema.FactorioConfigurationUpdate
import dev.adamko.kafkatorio.events.schema.FactorioEvent
import dev.adamko.kafkatorio.events.schema.FactorioEventUpdatePacket
import dev.adamko.kafkatorio.events.schema.FactorioPrototypes
import dev.adamko.kafkatorio.events.schema.KafkatorioPacket
import dev.adamko.kafkatorio.events.schema.MapChunkUpdate
import dev.adamko.kafkatorio.events.schema.PlayerUpdate
import dev.adamko.kafkatorio.schema2.ConfigurationUpdate
import dev.adamko.kafkatorio.schema2.ConsoleChatUpdate
import dev.adamko.kafkatorio.schema2.ConsoleCommandUpdate
import dev.adamko.kafkatorio.schema2.EntityUpdate
import dev.adamko.kafkatorio.schema2.KafkatorioPacket2
import dev.adamko.kafkatorio.schema2.MapChunkUpdate
import dev.adamko.kafkatorio.schema2.PlayerUpdate
import dev.adamko.kafkatorio.schema2.PrototypesUpdate
import dev.adamko.kafkatorio.schema2.SurfaceUpdate
import io.kvision.redux.ReduxStore
import kotlinx.serialization.decodeFromString
import org.w3c.dom.MessageEvent
@ -35,26 +36,23 @@ class WebsocketService(
println(data.replace('\n', ' '))
when (val event: KafkatorioPacket = jsonMapper.decodeFromString(data)) {
is FactorioEvent -> {
}
is FactorioEventUpdatePacket -> {
when (val eventUpdate = event.update) {
is PlayerUpdate -> reduxStore.dispatch(FactorioUpdate.Player(event.tick, eventUpdate))
val packet: KafkatorioPacket2 = jsonMapper.decodeFromString(data)
when (val packetData = packet.data) {
is PlayerUpdate -> reduxStore.dispatch(FactorioUpdate.Player(packet.tick, packetData))
is ConfigurationUpdate,
is ConsoleChatUpdate,
is ConsoleCommandUpdate,
is PrototypesUpdate,
is SurfaceUpdate,
is EntityUpdate,
is MapChunkUpdate -> {
}
}
}
is FactorioConfigurationUpdate,
is FactorioPrototypes -> {
// to be continued...
}
}
} else {
println(data)
}
}
}