lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' of github.com:adamko-dev/kafkatorio

Browse source
This commit is contained in:
Adam Semenenko 2022-06-27 22:04:05 +01:00
commit 5ac50e8412
73 changed files with 2328 additions and 1830 deletions
Download patch file Download diff file Expand all files Collapse all files

2
build.gradle.kts
View file

@ -12,7 +12,7 @@ plugins {
}
group = "dev.adamko.kafkatorio"
version = "0.6.0"
version = "0.6.1"
val licenseFile: RegularFile by extra(layout.projectDirectory.file("LICENSE"))

4
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPortalPublishClient.kt → buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPortalPublishClient.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.factoriomod
package dev.adamko.kafkatorio.factoriomod.portal
import io.ktor.client.HttpClient
import io.ktor.client.engine.cio.CIO
@ -98,7 +98,7 @@ class FactorioModPortalPublishClient(
require(response.status.isSuccess()) { "init upload request failed" }
return when (initUploadResponse) {
is Failure -> error(initUploadResponse)
is Failure -> error(initUploadResponse)
is InitUploadResponse.Success -> initUploadResponse
}
}

5
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/FactorioModPublishTask.kt → buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/FactorioModPublishTask.kt
View file

@ -1,5 +1,6 @@
package dev.adamko.kafkatorio.factoriomod
package dev.adamko.kafkatorio.factoriomod.portal
import dev.adamko.kafkatorio.factoriomod.FactorioMod
import io.ktor.http.URLBuilder
import io.ktor.http.takeFrom
import javax.inject.Inject
@ -59,7 +60,7 @@ abstract class FactorioModPublishTask @Inject constructor(
init {
group = "factorio"
group = FactorioMod.TASK_GROUP
}

2
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/model.kt → buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/portal/model.kt
View file

@ -1,4 +1,4 @@
package dev.adamko.kafkatorio.factoriomod
package dev.adamko.kafkatorio.factoriomod.portal
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable

12
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/factoriomod/tasks.kt Normal file
View file

@ -0,0 +1,12 @@
package dev.adamko.kafkatorio.factoriomod
object FactorioMod {
const val TASK_GROUP = "factorioMod"
/** Lifecycle task for publishing the mod locally */
const val PUBLISH_MOD_LOCAL_TASK_NAME = "publishModLocal"
const val PUBLISH_MOD_TASK_NAME = "publishMod"
}

8
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts
View file

@ -12,16 +12,8 @@ val projectJvmTarget = "11"
dependencies {
// implementation(platform("org.jetbrains.kotlin:kotlin-bom"))
// testImplementation(platform("org.junit:junit-bom:${Versions.junit}"))
testImplementation("org.junit.jupiter:junit-jupiter")
// testRuntimeOnly("org.junit.platform:junit-platform-launcher") {
// because("Only needed to run tests in a version of IntelliJ IDEA that bundles older versions")
// }
// testImplementation(platform("io.kotest:kotest-bom:${Versions.kotest}"))
testImplementation("io.kotest:kotest-runner-junit5")
testImplementation("io.kotest:kotest-assertions-core")
testImplementation("io.kotest:kotest-property")

2
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypescriptTask.kt → buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/GenerateTypeScriptTask.kt
View file

@ -8,7 +8,7 @@ import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.TaskAction
abstract class GenerateTypescriptTask @Inject constructor(
abstract class GenerateTypeScriptTask @Inject constructor(
private val fileOps: FileSystemOperations,
) : JavaExec() {

8
gradle/libs.versions.toml
View file

@ -35,6 +35,8 @@ ktor = "2.0.2"
hoplite = "2.1.5"
simpleSyslogParser = "0.0.6"
kotkaStreams = "dev-SNAPSHOT"
## test ##
@ -156,10 +158,12 @@ ktor-serverCallLogging = { group = "io.ktor", name = "ktor-server-call-l
ktor-serverCallId = { group = "io.ktor", name = "ktor-server-call-id" }
ktor-network = { group = "io.ktor", name = "ktor-network" }
ktor-networkTls = { group = "io.ktor", name = "ktor-network-tls" }
ktor-test-clientMock = { group = "io.ktor", name = "ktor-client-mock" }
ktor-test-serverHost = { group = "io.ktor", name = "ktor-server-test-host" }
### jackson ##
#jackson-bom = { group = " com.fasterxml.jackson.module", name = ":jackson-module-kotlin", version = "2.13.+" }
#jackson-kotlin = { group = " com.fasterxml.jackson.module", name = ":jackson-module-kotlin", version = "2.13.+" }
@ -205,6 +209,10 @@ hoplite-core = { group = "com.sksamuel.hoplite", name = "hoplite-core",
hoplite-datetime = { group = "com.sksamuel.hoplite", name = "hoplite-datetime", version.ref = "hoplite" }
hoplite-yaml = { group = "com.sksamuel.hoplite", name = "hoplite-yaml", version.ref = "hoplite" }
simpleSyslogParser = { group = "com.github.palindromicity", name = "simple-syslog", version.ref = "simpleSyslogParser" }
#############
### test ###
#############

21
modules/events-mod/build.gradle.kts
View file

@ -1,4 +1,4 @@
import dev.adamko.kafkatorio.factoriomod.FactorioModPublishTask
import dev.adamko.kafkatorio.factoriomod.portal.FactorioModPublishTask
import dev.adamko.kafkatorio.gradle.asConsumer
import dev.adamko.kafkatorio.gradle.asProvider
import dev.adamko.kafkatorio.gradle.dropDirectory
@ -39,11 +39,11 @@ val projectTokens: MapProperty<String, String> =
?: error("error getting projectTokens")
//val projectTokens: MapProperty<String, String> by rootProject.extra
val projectTokensX = projectTokens.apply {
val projectTokensX1 = projectTokens.apply {
put("mod.name", modName)
put("mod.title", "Kafkatorio Events")
put("mod.description", modDescription)
put("factorio.version", modFactorioCompatibility.get())
put("factorio.version", modFactorioCompatibility)
}
val tsSrcDir: Directory = layout.projectDirectory.dir("src/main/typescript")
@ -82,7 +82,7 @@ val installEventsTsSchema by tasks.registering(Sync::class) {
dependsOn(typescriptEventsSchema)
// dependsOn(fixLink)
val outputDir = layout.projectDirectory.dir("src/main/typescript/generated/kafkatorio-schema")
val outputDir = layout.projectDirectory.dir("src/main/typescript/generated/")
outputs.dir(outputDir)
from(
@ -107,16 +107,16 @@ val installEventsTsSchema by tasks.registering(Sync::class) {
val zipNameProvider = provider { distributionZipName }
tasks.distZip {
val projectTokensXX = projectTokensX
val projectTokensXX22 = projectTokensX1
inputs.property("zipNameProvider", zipNameProvider)
inputs.property("projectTokens", projectTokensXX)
inputs.property("projectTokens", projectTokensXX22)
archiveFileName.set(zipNameProvider)
}
val publishMod by tasks.registering(FactorioModPublishTask::class) {
val publishModToPortal by tasks.registering(FactorioModPublishTask::class) {
dependsOn(tasks.check)
distributionZip.set(tasks.distZip.flatMap { it.archiveFile })
@ -131,6 +131,7 @@ distributions {
main {
distributionBaseName.set(modName)
val tokens333: MutableMap<String, String> = projectTokensX1.get()
contents {
from(layout.projectDirectory.dir("src/main/resources/mod-data")) {
@ -139,7 +140,7 @@ distributions {
from(licenseFile)
from(typescriptToLua.map { it.outputDirectory })
filesNotMatching("**/*.png") {
filter<ReplaceTokens>("tokens" to projectTokens.get())
filter<ReplaceTokens>("tokens" to tokens333)
}
includeEmptyDirs = false
exclude {
@ -154,8 +155,8 @@ distributions {
tasks.withType<Zip>().configureEach {
val projectTokensXX = projectTokensX
inputs.property("projectTokens", projectTokensXX)
val projectTokensXX444 = projectTokensX1
inputs.property("projectTokens", projectTokensXX444)
}

2
modules/events-mod/package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "kafkatorio-events-mod",
"version": "0.6.0",
"version": "0.6.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

14
modules/events-mod/package.json
View file

@ -1,6 +1,6 @@
{
"name": "kafkatorio-events-mod",
"version": "0.6.0",
"version": "0.6.1",
"private": true,
"scripts": {
"build": "tstl",
@ -9,7 +9,8 @@
"engines": {
"node": "~14.19.1"
},
"devDependencies": {},
"devDependencies": {
},
"dependencies": {
"lua-types": "^2.11.0",
"typescript-to-lua": "1.6.0",
@ -17,7 +18,10 @@
"typescript": "4.7.4"
},
"license": "UNLICENSED",
"peerDependencies": {},
"optionalDependencies": {},
"bundledDependencies": []
"peerDependencies": {
},
"optionalDependencies": {
},
"bundledDependencies": [
]
}

2564
modules/events-mod/src/main/resources/kafkatorio-logo.svg
View file

File diff suppressed because it is too large Load diff
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 991 KiB

After

Width:  |  Height:  |  Size: 1.1 MiB

Before After
Before After

BIN
modules/events-mod/src/main/resources/mod-data/thumbnail.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 8 KiB

Before After
Before After

20
modules/events-mod/src/main/typescript/control.ts
View file

@ -1,23 +1,11 @@
import {emitConfigurationUpdate} from "./main/config-update/handleConfigurationUpdate";
import {emitPrototypes} from "./main/config-update/prototypes";
import {initGlobal} from "./main/global-init";
import KafkatorioSettings from "./main/settings/KafkatorioSettings";
import {emitPrototypes} from "./main/events/prototypeUpdates";
require("./main/events/registerHandlers")
require("./main/events/onMapChunkEventListeners")
require("./main/events/onPlayerEventListeners")
require("./main/events/EntityEvents")
require("./main/events/mapChunkUpdates")
require("./main/events/playerUpdates")
require("./main/events/entityUpdates")
require("./main/commands/kafkatorioCommands")
require("./main/cache/cacheChecker")
script.on_configuration_changed((data: ConfigurationChangedData) => {
// runs whenever the version of the base game, or a mod, changes
initGlobal(true)
// KafkatorioSettings.loadSettings()
emitConfigurationUpdate(data)
emitPrototypes()
})
script.on_init(() => {

12
modules/events-mod/src/main/typescript/generated/kafkatorio-schema/kafkatorio-schema.ts → modules/events-mod/src/main/typescript/generated/kafkatorio-schema.ts
View file

@ -13,6 +13,7 @@ export type KafkatorioPacketData =
| KafkatorioPacketData.ConsoleChatUpdate
| KafkatorioPacketData.ConsoleCommandUpdate
| KafkatorioPacketData.EntityUpdate
| KafkatorioPacketData.Error
| KafkatorioPacketData.MapChunkUpdate
| KafkatorioPacketData.PlayerUpdate
| KafkatorioPacketData.PrototypesUpdate
@ -28,6 +29,7 @@ export namespace KafkatorioPacketData {
EntityUpdate = "kafkatorio.packet.keyed.EntityUpdate",
MapChunkUpdate = "kafkatorio.packet.keyed.MapChunkUpdate",
PlayerUpdate = "kafkatorio.packet.keyed.PlayerUpdate",
Error = "kafkatorio.packet.KafkatorioPacketData.Error",
}
export interface ConfigurationUpdate {
@ -115,6 +117,12 @@ export namespace KafkatorioPacketData {
disconnectReason?: string | null;
isRemoved?: boolean | null;
}
export interface Error {
type: KafkatorioPacketData.Type.Error;
message?: string | null;
rawValue?: string | null;
}
}
export type UInt = uint;
@ -240,12 +248,12 @@ export type List = any;
export type PrototypeKey = Int;
export interface MinedProduct {
type: Type;
type: MinedProductType;
name: PrototypeName;
amount: Double | null;
}
export enum Type {
export enum MinedProductType {
item = "item",
fluid = "fluid",
}

3
modules/events-mod/src/main/typescript/main/PacketEmitter.ts
View file

@ -1,9 +1,10 @@
import {
KafkatorioPacket,
KafkatorioPacketData
} from "../generated/kafkatorio-schema/kafkatorio-schema";
} from "../generated/kafkatorio-schema";
import {KafkatorioKeyedPacketData} from "./types";
export class KafkatorioPacketEmitter {
emitInstantPacket<DATA extends KafkatorioPacketData>(

27
modules/events-mod/src/main/typescript/main/cache/EventDataCache.ts vendored
View file

@ -1,5 +1,6 @@
import KafkatorioSettings from "../settings/KafkatorioSettings";
import {KafkatorioKeyedPacketData} from "../types";
import packetEmitter from "../PacketEmitter";
export namespace EventUpdates {
@ -71,6 +72,10 @@ export namespace EventUpdates {
if (expirationDurationTicks != undefined) {
entry.expirationDurationTicks = expirationDurationTicks
}
if (entry.expirationDurationTicks != undefined && entry.expirationDurationTicks <= 0) {
this.extractAndEmitExpiredPackets()
}
}
@ -90,8 +95,9 @@ export namespace EventUpdates {
}
/** Removes all expired entries from the cache, and return them. */
// use in/out v4.7.0 https://github.com/microsoft/TypeScript/pull/48240
public extractExpired(): Array<KafkatorioKeyedPacketData> {
private extractExpired(): Array<KafkatorioKeyedPacketData> {
let countExpired = 0
let countTotal = 0
const expiredData: Array<KafkatorioKeyedPacketData> = []
@ -114,6 +120,25 @@ export namespace EventUpdates {
}
/**
* Removes and emits all expired packets.
*
* @return the number of emitted packets
*/
public extractAndEmitExpiredPackets() {
const cachedEvents: Array<KafkatorioKeyedPacketData> = this.extractExpired()
if (cachedEvents.length > 0) {
log(`[EventDataCache] emitting ${cachedEvents.length} events on tick ${game.tick}`)
for (const event of cachedEvents) {
packetEmitter.emitKeyedPacket(event)
}
}
return cachedEvents.length
}
private getCacheEntry<PACKET extends KafkatorioKeyedPacketData>(
key: EventUpdates.PacketKey<PACKET>,
type: EventUpdates.PacketType<PACKET>,

16
modules/events-mod/src/main/typescript/main/cache/cacheChecker.ts vendored
View file

@ -1,16 +0,0 @@
import EventCacheService from "./EventDataCache";
import packetEmitter from "../PacketEmitter";
import {KafkatorioKeyedPacketData} from "../types";
/** Every 0.1 seconds */
script.on_nth_tick(6, (nthTick: NthTickEventData) => {
const cachedEvents: Array<KafkatorioKeyedPacketData> = EventCacheService.extractExpired()
if (cachedEvents.length > 0) {
log(`nth tick ${nthTick.tick} has ${cachedEvents.length} events`)
for (const event of cachedEvents) {
packetEmitter.emitKeyedPacket(event)
}
}
})

9
modules/events-mod/src/main/typescript/main/commands/kafkatorioCommands.ts
View file

@ -1,7 +1,6 @@
import {emitPrototypes} from "../config-update/prototypes";
import {initGlobal} from "../global-init";
import floor = math.floor;
import {Queue} from "../queue/queue";
import {emitPrototypes} from "../events/prototypeUpdates";
commands.add_command(
@ -32,8 +31,8 @@ commands.add_command(
let player = game.players[e.player_index]
const chunkPosition: ChunkPosition = {
x: floor(player.position.x / 32),
y: floor(player.position.y / 32),
x: math.floor(player.position.x / 32),
y: math.floor(player.position.y / 32),
}
const chunkXMin = chunkPosition.x - radius
@ -56,7 +55,7 @@ commands.add_command(
position: {x: chunk.x, y: chunk.y},
area: chunk.area,
surface: surface,
tick: e.tick
tick: e.tick,
}
Queue.enqueue(

20
modules/events-mod/src/main/typescript/main/config-update/handleConfigurationUpdate.ts → modules/events-mod/src/main/typescript/main/events/configurationUpdate.ts
View file

@ -3,11 +3,23 @@ import {
ConfigurationUpdateGameData,
ConfigurationUpdateModData,
KafkatorioPacketData
} from "../../generated/kafkatorio-schema/kafkatorio-schema";
} from "../../generated/kafkatorio-schema";
import {initGlobal} from "../global-init";
import {emitPrototypes} from "./prototypeUpdates";
import Type = KafkatorioPacketData.Type;
import ConfigurationUpdate = KafkatorioPacketData.ConfigurationUpdate;
export function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
script.on_configuration_changed((data: ConfigurationChangedData) => {
// runs whenever the version of the base game, or a mod, changes
initGlobal(true)
// KafkatorioSettings.loadSettings()
emitConfigurationUpdate(data)
emitPrototypes()
})
function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
const configUpdateData: ConfigurationUpdate = {
type: Type.ConfigurationUpdate,
@ -20,6 +32,7 @@ export function emitConfigurationUpdate(changeData: ConfigurationChangedData) {
PacketEmitter.emitInstantPacket(configUpdateData)
}
function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdateGameData {
return {
oldVersion: changeData.old_version ?? null,
@ -27,6 +40,7 @@ function factorioData(changeData: ConfigurationChangedData): ConfigurationUpdate
}
}
function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] {
let namesOfChangedMods: string[] = Object.keys(e.mod_changes)
@ -38,7 +52,7 @@ function allMods(e: ConfigurationChangedData): ConfigurationUpdateModData[] {
return modNames.map((modName) => {
let currentVer = script.active_mods[modName] ?? e.mod_changes[modName].new_version ?? null
let currentVer = script.active_mods[modName] ?? e.mod_changes[modName]?.new_version ?? null
let previousVer = e.mod_changes[modName]?.old_version ?? null
return {

4
modules/events-mod/src/main/typescript/main/events/onConsoleEventHandlers.ts → modules/events-mod/src/main/typescript/main/events/consoleUpdates.ts
View file

@ -1,7 +1,8 @@
import PacketEmitter from "../PacketEmitter";
import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import {KafkatorioPacketData} from "../../generated/kafkatorio-schema";
import Type = KafkatorioPacketData.Type;
script.on_event(
defines.events.on_console_chat,
(event: OnConsoleChatEvent) => {
@ -15,6 +16,7 @@ script.on_event(
}
)
script.on_event(
defines.events.on_console_command,
(event: OnConsoleCommandEvent) => {

11
modules/events-mod/src/main/typescript/main/events/converters.ts
View file

@ -1,5 +1,5 @@
import {EventName} from "../types";
import {Colour, MapChunkPosition} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import {Colour, MapChunkPosition} from "../../generated/kafkatorio-schema";
export namespace Converters {
@ -57,4 +57,13 @@ export namespace Converters {
return [position.x, position.y]
}
export function convertCollisionMaskToNames(cm: CollisionMask): string[] {
let masks: string[] = []
for (let [name] of pairs(cm)) {
masks.push(name)
}
return masks
}
}

2
modules/events-mod/src/main/typescript/main/events/EntityEvents.ts → modules/events-mod/src/main/typescript/main/events/entityUpdates.ts
View file

@ -3,7 +3,7 @@ import EventDataCache from "../cache/EventDataCache";
import {
EntityUpdateKey,
KafkatorioPacketData
} from "../../generated/kafkatorio-schema/kafkatorio-schema";
} from "../../generated/kafkatorio-schema";
script.on_event(defines.events.on_built_entity, (e: OnBuiltEntityEvent) => {

2
modules/events-mod/src/main/typescript/main/events/onMapChunkEventListeners.ts → modules/events-mod/src/main/typescript/main/events/mapChunkUpdates.ts
View file

@ -4,7 +4,7 @@ import EventUpdatesManager, {EventUpdates} from "../cache/EventDataCache";
import {
KafkatorioPacketData,
MapChunkPosition
} from "../../generated/kafkatorio-schema/kafkatorio-schema";
} from "../../generated/kafkatorio-schema";
import CacheKey = EventUpdates.PacketKey;
import Type = KafkatorioPacketData.Type;

35
modules/events-mod/src/main/typescript/main/events/onPlayerEventListeners.ts → modules/events-mod/src/main/typescript/main/events/playerUpdates.ts
View file

@ -1,7 +1,11 @@
import {Converters} from "./converters";
import EventUpdatesManager from "../cache/EventDataCache";
import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import {
KafkatorioPacketData,
PlayerUpdateKey
} from "../../generated/kafkatorio-schema";
import Type = KafkatorioPacketData.Type;
import packetEmitter from "../PacketEmitter";
type PlayerUpdater = (player: LuaPlayer, data: KafkatorioPacketData.PlayerUpdate) => void
@ -44,7 +48,7 @@ function playerUpdateThrottle(
data.events[eventName] ??= []
data.events[eventName].push(event.tick)
},
expirationDurationTicks
expirationDurationTicks,
)
}
@ -183,13 +187,26 @@ script.on_event(
defines.events.on_pre_player_left_game,
(event: OnPrePlayerLeftGameEvent) => {
log(`on_pre_player_left_game ${event.tick} ${event.name}`)
playerUpdateImmediate(
event,
(player, data) => {
data.disconnectReason = disconnectReasons.get(event.reason)
playerOnlineInfo(player, data)
}
)
const playerUpdateKey: PlayerUpdateKey = {
index: event.player_index
}
const playerUpdate: KafkatorioPacketData.PlayerUpdate = {
type: KafkatorioPacketData.Type.PlayerUpdate,
key: playerUpdateKey,
disconnectReason: disconnectReasons.get(event.reason)
}
packetEmitter.emitKeyedPacket(playerUpdate)
// playerUpdateImmediate(
// event,
// (player, data) => {
// data.disconnectReason = disconnectReasons.get(event.reason)
// playerOnlineInfo(player, data)
// }
// )
}
)

26
modules/events-mod/src/main/typescript/main/config-update/prototypes.ts → modules/events-mod/src/main/typescript/main/events/prototypeUpdates.ts
View file

@ -1,24 +1,21 @@
import {Converters} from "../events/converters";
import PacketEmitter from "../PacketEmitter";
import {
FactorioPrototype,
KafkatorioPacketData
} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import Type = KafkatorioPacketData.Type;
import {FactorioPrototype, KafkatorioPacketData} from "../../generated/kafkatorio-schema";
import {Converters} from "./converters";
import PrototypesUpdate = KafkatorioPacketData.PrototypesUpdate;
export function emitPrototypes() {
const data: PrototypesUpdate = {
prototypes: prototypes(),
type: Type.PrototypesUpdate,
prototypes: prototypeUpdates(),
type: KafkatorioPacketData.Type.PrototypesUpdate,
}
PacketEmitter.emitInstantPacket(data)
}
function prototypes(): FactorioPrototype[] {
function prototypeUpdates(): FactorioPrototype[] {
let prototypes: FactorioPrototype[] = []
prototypes.push(...getMapTilePrototypes())
@ -26,6 +23,7 @@ function prototypes(): FactorioPrototype[] {
return prototypes
}
function getMapTilePrototypes(): FactorioPrototype.MapTile[] {
let tiles: FactorioPrototype.MapTile[] = []
for (let [, tile] of game.tile_prototypes) {
@ -35,7 +33,7 @@ function getMapTilePrototypes(): FactorioPrototype.MapTile[] {
name: tile.name,
order: tile.order,
layer: tile.layer,
collisionMasks: convertCollisionMaskToNames(tile.collision_mask),
collisionMasks: Converters.convertCollisionMaskToNames(tile.collision_mask),
mapColour: Converters.mapColour(tile.map_color),
canBeMined: tile.mineable_properties.minable,
})
@ -60,11 +58,3 @@ function getMapTilePrototypes(): FactorioPrototype.MapTile[] {
// }
// return tiles
// }
function convertCollisionMaskToNames(cm: CollisionMask): string[] {
let masks: string[] = []
for (let [name] of pairs(cm)) {
masks.push(name)
}
return masks
}

50
modules/events-mod/src/main/typescript/main/events/registerHandlers.ts
View file

@ -1,34 +1,11 @@
import {Queue} from "../queue/queue";
import {isEventType} from "./eventTypeCheck";
import {Converters} from "./converters";
import KafkatorioSettings from "../settings/KafkatorioSettings";
import {handleChunkGeneratedEvent} from "./onMapChunkEventListeners";
import {handleSurfaceUpdate} from "./handlers";
import {handleChunkGeneratedEvent} from "./mapChunkUpdates";
import {handleSurfaceUpdate} from "./surfaceUpdates";
import EventCacheService from "../cache/EventDataCache";
// script.on_event(
// defines.events.on_player_mined_entity,
// (e: OnPlayerMinedEntityEvent) => {
// let eventName = Converters.eventNameString(e.name)
// // handlePlayerUpdate(e.tick, eventName, e.player_index)
// handleEntityUpdate(e.tick, eventName, e.entity)
// }
// )
// script.on_event(
// defines.events.on_player_joined_game,
// (e: OnPlayerJoinedGameEvent) => {
// handlePlayerUpdate(e.tick, mapEventIdToName.get(e.name), e.player_index)
// }
// )
// script.on_event(
// defines.events.on_player_changed_position,
// (e: OnPlayerChangedPositionEvent) => {
// handlePlayerUpdate(e.tick, mapEventIdToName.get(e.name), e.player_index)
// }
// )
script.on_event(
defines.events.on_tick,
(event: OnTickEvent) => {
@ -38,18 +15,26 @@ script.on_event(
}
}
if (event.tick % 6 == 0) {
// Every 0.1 seconds
const emittedCount = EventCacheService.extractAndEmitExpiredPackets()
if (emittedCount > 0) {
log(`[on_tick:${event.tick}] emitted ${emittedCount} events`)
}
}
if (event.tick % 7 == 0) {
const events: EventData[] = Queue.dequeueValues(1)
if (events.length > 0) {
log(`[${event.tick}] dequed ${events.length} events, current size: ${Queue.size()}`)
log(`[on_tick:${event.tick}] dequeued ${events.length} events, current size: ${Queue.size()}`)
let i = 1
for (const event of events) {
if (isEventType(event, defines.events.on_chunk_generated)) {
let eName = Converters.eventNameString(event.name)
log(`[${event.tick}] dequed event ${eName}, delay ${i}`)
log(`[on_tick:${event.tick}] dequed event ${eName}, delay ${i}`)
handleChunkGeneratedEvent(event, i)
i += 1
}
@ -58,12 +43,3 @@ script.on_event(
}
}
)
script.on_event(
defines.events.on_runtime_mod_setting_changed,
(e: OnRuntimeModSettingChangedEvent) => {
print(`Settings changed ${e.setting_type}, ${e.player_index}, ${e.setting}, ${e.mod_name}`)
// KafkatorioSettings.loadSettings()
}
)

7
modules/events-mod/src/main/typescript/main/events/settingsUpdate.ts Normal file
View file

@ -0,0 +1,7 @@
script.on_event(
defines.events.on_runtime_mod_setting_changed,
(e: OnRuntimeModSettingChangedEvent) => {
print(`Settings changed ${e.setting_type}, ${e.player_index}, ${e.setting}, ${e.mod_name}`)
// KafkatorioSettings.loadSettings()
}
)

2
modules/events-mod/src/main/typescript/main/events/handlers.ts → modules/events-mod/src/main/typescript/main/events/surfaceUpdates.ts
View file

@ -1,5 +1,5 @@
import PacketEmitter from "../PacketEmitter";
import {KafkatorioPacketData} from "../../generated/kafkatorio-schema/kafkatorio-schema";
import {KafkatorioPacketData} from "../../generated/kafkatorio-schema";
import Type = KafkatorioPacketData.Type;

23
modules/events-mod/src/main/typescript/main/settings/KafkatorioSettings.ts
View file

@ -11,35 +11,12 @@ export class KafkatorioSettingsConfig {
}
// private eventCacheExpirationTicks: Record<KafkatorioKeyedPacketTypes, double> = {
// "kafkatorio.packet.keyed.PlayerUpdate":
// KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.PlayerUpdate],
// "kafkatorio.packet.keyed.MapChunkUpdate":
// KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.MapChunkUpdate],
// "kafkatorio.packet.keyed.EntityUpdate":
// KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds[KafkatorioPacketData.Type.EntityUpdate],
// }
public getEventCacheExpirationTicks(packet: KafkatorioKeyedPacketData): uint {
const settingName = KafkatorioSettingsConfig.cacheDurationSettingName(packet.type)
const seconds: double = settings.global[settingName].value as double
return seconds * 60 // convert ticks to seconds
}
// private static types: KafkatorioKeyedPacketTypes[] = [
// KafkatorioPacketData.Type.PlayerUpdate,
// KafkatorioPacketData.Type.MapChunkUpdate,
// KafkatorioPacketData.Type.EntityUpdate,
// ]
// public loadSettings(): void {
// for (const type of KafkatorioSettingsConfig.types) {
// const settingName = KafkatorioSettingsConfig.cacheDurationSettingName(type)
// const seconds: double = settings.global[settingName].value as double
// this.eventCacheExpirationTicks[type] = seconds * 60 // convert ticks to seconds
// }
// }
public initialiseSettings(data: Data): void {
for (const [type,] of pairs(KafkatorioSettingsConfig.eventCacheExpirationDefaultSeconds)) {

2
modules/events-mod/src/main/typescript/main/types.d.ts vendored
View file

@ -1,4 +1,4 @@
import {KafkatorioPacketData} from "../generated/kafkatorio-schema/kafkatorio-schema";
import {KafkatorioPacketData} from "../generated/kafkatorio-schema";
export type EventName = keyof typeof defines.events

79
modules/events-schema/build.gradle.kts
View file

@ -1,6 +1,6 @@
import dev.adamko.kafkatorio.gradle.asProvider
import dev.adamko.kafkatorio.gradle.typescriptAttributes
import dev.adamko.kafkatorio.task.GenerateTypescriptTask
import dev.adamko.kafkatorio.task.GenerateTypeScriptTask
plugins {
@ -14,25 +14,8 @@ plugins {
kotlin {
js(IR) {
// binaries.executable()
browser {}
// browser{} causes this error:
// Cannot load "webpack", it is not registered!
// Perhaps you are missing some plugin?
// Cannot load "sourcemap", it is not registered!
// Perhaps you are missing some plugin?
// Server start failed on port 9876: Error: No provider for "framework:mocha"! (Resolving: framework:mocha)
// browser {
// runTask {
// sourceMaps = true
// }
// testTask {
// useKarma {
// useChromeHeadless()
// }
// }
// }
// nodejs()
// this is a library - don't set binaries.executable()
}
jvm {
val main by compilations.getting {
@ -75,9 +58,6 @@ kotlin {
implementation(project.dependencies.platform(libs.kotest.bom))
implementation("io.kotest:kotest-framework-engine")
// implementation("io.kotest:kotest-assertions-core")
// implementation("io.kotest:kotest-property")
// implementation("io.kotest:kotest-assertions-json")
implementation(libs.kotest.core)
implementation(libs.kotest.datatest)
implementation(libs.kotest.prop)
@ -88,10 +68,7 @@ kotlin {
val jvmMain by getting {
dependencies {
implementation("com.github.aSemy:ts-generator:v1.2.1")
// implementation("com.github.ntrrgc:ts-generator:1.1.2")
implementation(kotlin("reflect"))
// implementation("com.github.adamko-dev:kotlinx-serialization-typescript-generator:0.0.5")
}
}
@ -114,7 +91,7 @@ val mainCompilation: Provider<FileCollection> =
kotlin.jvm().compilations.named("main").map { it.runtimeDependencyFiles }
val generateTypescript by tasks.registering(GenerateTypescriptTask::class) {
val generateTypeScript by tasks.registering(GenerateTypeScriptTask::class) {
dependsOn(jvmJar)
classpath(
jvmJar,
@ -125,68 +102,26 @@ val generateTypescript by tasks.registering(GenerateTypescriptTask::class) {
args(temporaryDir.canonicalPath)
}
//val generateTypescript2 by tasks.registering(JavaExec::class) {
// group = "kt2ts"
//
// dependsOn(jvmJar)
//
// inputs.files(mainCompilation)
//
// classpath(
// jvmJar,
// mainCompilation,
// )
// mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2ts2Kt")
//
// args(temporaryDir)
//
// val buildOutput = layout.buildDirectory.dir("generated/typescript")
// outputs.dir(buildOutput)
//
// doFirst {
// delete(temporaryDir)
// mkdir(temporaryDir)
// }
//
// doLast {
// sync {
// from(temporaryDir)
// into(buildOutput)
// include("**/*.ts")
// }
// }
//}
val schemaTsDistributionName: Provider<String> = providers.provider {
"${rootProject.name}-${project.name}"
}
val generateTypescriptOutputFiles: Provider<FileTree> =
generateTypescript.map { it.outputs.files.asFileTree }
val generateTypeScriptOutputFiles: Provider<FileTree> =
generateTypeScript.map { it.outputs.files.asFileTree }
val schemaTs by distributions.registering {
distributionBaseName.set(schemaTsDistributionName)
contents {
from(generateTypescriptOutputFiles)
from(generateTypeScriptOutputFiles)
}
}
val schemaTsZipTask: TaskProvider<Zip> = tasks.named<Zip>("${schemaTs.name}DistZip")
val schemaTsZipTaskArchiveFile: Provider<RegularFile> = schemaTsZipTask.flatMap { it.archiveFile }
val typescriptModelGenerated: Configuration by configurations.creating {
val typeScriptModelGenerated: Configuration by configurations.creating {
asProvider()
typescriptAttributes(objects)
outgoing.artifact(schemaTsZipTaskArchiveFile)
}
//tasks.matching {
// it.name == "jsGenerateExternalsIntegrated"
//}.configureEach {
// @Suppress("UnstableApiUsage")
// notCompatibleWithConfigurationCache("try to prevent 'Projects must be configuring' error")
//// outputs.upToDateWhen { true }
// dependsOn(":kotlinNpmInstall")
//}

2
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/common/resources.kt
View file

@ -5,6 +5,7 @@ import kotlinx.serialization.Serializable
@Serializable
@SerialName("kafkatorio.resource.MinedProduct")
data class MinedProduct(
/** `item` or `fluid`. */
val type: Type,
@ -30,6 +31,7 @@ data class MinedProduct(
) {
@Serializable
@SerialName("kafkatorio.resource.MinedProductType")
enum class Type {
@SerialName("item")
ITEM,

11
modules/events-schema/src/commonMain/kotlin/dev/adamko/kafkatorio/schema/packets/kafkatorioPacket.kt
View file

@ -16,4 +16,13 @@ data class KafkatorioPacket(
@Serializable
@SerialName("kafkatorio.packet.KafkatorioPacketData")
sealed class KafkatorioPacketData
sealed class KafkatorioPacketData {
@Serializable
@SerialName("kafkatorio.packet.KafkatorioPacketData.Error")
data class Error(
val message: String? = null,
val rawValue: String? = null,
) : KafkatorioPacketData()
}

15
modules/events-server/build.gradle.kts
View file

@ -1,12 +1,14 @@
import dev.adamko.kafkatorio.task.KafkaConsumerGroupsTask
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
dev.adamko.kafkatorio.lang.`kotlin-jvm`
kotlin("plugin.serialization")
application
}
description = """
Receive raw Factorio Events from Kafka and process them into targeted topics or data formats.
@ -17,6 +19,7 @@ val projectId: String by project.extra
val kafkaStateDir: Directory = layout.projectDirectory.dir("kafka-state")
dependencies {
implementation(platform(projects.modules.versionsPlatform))
@ -42,6 +45,11 @@ dependencies {
implementation(libs.okio.core)
implementation(libs.ktor.network)
implementation(libs.ktor.networkTls)
implementation(libs.simpleSyslogParser)
implementation(libs.kafka.kotkaStreams) {
isChanging = true
}
@ -49,9 +57,6 @@ dependencies {
implementation(libs.bundles.hoplite)
}
//configurations.all {
// resolutionStrategy.cacheDynamicVersionsFor(1, TimeUnit.MINUTES)
//}
application {
mainClass.set("dev.adamko.kafkatorio.processor.EventProcessorKt")
@ -93,7 +98,3 @@ idea {
excludeDirs = excludeDirs + kafkaStateDir.asFile
}
}
//tasks.withType<KotlinNpmInstallTask>().all {
// notCompatibleWithConfigurationCache("error: Projects must be configuring")
//}

2
modules/events-server/docker/.env
View file

@ -1,3 +1,3 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
KAFKATORIO_VERSION=0.6.0
KAFKATORIO_VERSION=0.6.1

6
modules/events-server/docker/docker-compose.yml
View file

@ -15,16 +15,16 @@ services:
environment:
KAFKA_BOOTSTRAP_SERVERS: "http://kafka:29092"
# KAFKA_STREAMS_STATE_DIR: "http://kafka:29092"
SERVER_PORT: "12081"
SERVER_PORT: "8080"
ports:
- "12080:12081" # http and ws
- "12080:8080" # http and ws
networks:
- "kafka-nw"
- "kafkatorio-nw"
- "traefik-nw"
labels:
- "traefik.enable=true"
- "traefik.http.services.events-server-kafkatorio.loadbalancer.server.port=12081"
# - "traefik.http.services.events-server-kafkatorio.loadbalancer.server.port=12081"
networks:
kafka-nw:

80
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/KafkatorioTopology.kt
View file

@ -1,6 +1,9 @@
package dev.adamko.kafkatorio.processor
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer
import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer
import dev.adamko.kafkatorio.processor.topology.broadcastToWebsocket
import dev.adamko.kafkatorio.processor.topology.colourMapChunks
@ -17,10 +20,17 @@ import kotlinx.coroutines.CoroutineName
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.cancel
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.flow.filter
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
import kotlinx.coroutines.suspendCancellableCoroutine
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.clients.producer.ProducerConfig
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.serialization.Serdes
import org.apache.kafka.streams.KafkaStreams
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.StreamsConfig
@ -29,12 +39,14 @@ import org.apache.kafka.streams.TopologyDescription
internal class KafkatorioTopology(
private val websocketServer: WebsocketServer,
private val appProps: ApplicationProperties,
private val websocketServer: WebsocketServer,
private val syslogServer: SyslogSocketServer,
) {
private val builderScope = CoroutineScope(CoroutineName("KafkaTopology"))
suspend fun start() = builderScope.launch {
splitPackets()
@ -42,8 +54,11 @@ internal class KafkatorioTopology(
saveTiles()
websocketBroadcaster()
syslogProducer()
}
private suspend fun splitPackets() {
val builder = StreamsBuilder()
val packets = factorioServerPacketStream(builder)
@ -54,18 +69,21 @@ internal class KafkatorioTopology(
launchTopology("splitPackets", topology)
}
private suspend fun groupTilesMapChunks() {
val builder = StreamsBuilder()
val topology = colourMapChunks(builder)
launchTopology("groupTilesMapChunks", topology)
}
private suspend fun saveTiles() {
val builder = StreamsBuilder()
val topology = saveMapTiles(builder, appProps.tileDir)
launchTopology("saveTiles", topology)
}
private suspend fun websocketBroadcaster() {
val builder = StreamsBuilder()
broadcastToWebsocket(websocketServer, builder)
@ -75,11 +93,65 @@ internal class KafkatorioTopology(
launchTopology(
"websocketBroadcaster", topology, mapOf(
StreamsConfig.COMMIT_INTERVAL_MS_CONFIG to "${1.seconds.inWholeMilliseconds}",
StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${10.milliseconds.inWholeMilliseconds}"
StreamsConfig.producerPrefix(ProducerConfig.LINGER_MS_CONFIG) to "${1.milliseconds.inWholeMilliseconds}"
)
)
}
private fun syslogProducer(
id: String = "syslog-producer"
) {
val props: Properties = appProps.kafkaStreamsConfig.toProperties()
val appId = props.compute(StreamsConfig.APPLICATION_ID_CONFIG) { _, v -> "$v.$id" } as String
props.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, appId)
val producer: KafkaProducer<String, String> = KafkaProducer(
props,
Serdes.String().serializer(),
Serdes.String().serializer(),
)
builderScope.coroutineContext.job.invokeOnCompletion {
producer.close()
}
syslogServer.messages
.filter { it.message?.startsWith("KafkatorioPacket:") == true }
.onEach { syslogMsg ->
val kafkatorioMessage = syslogMsg.message?.substringAfter("KafkatorioPacket:")
val record = ProducerRecord<String, String>(
TOPIC_SRC_SERVER_LOG,
"syslog-test",
kafkatorioMessage,
)
val headers = SyslogFieldKeys.values()
.filter { "HEADER_" in it.name }
.mapNotNull { headerKey ->
val headerValue = syslogMsg.src[headerKey.field]
when {
headerValue.isNullOrBlank() -> null
else -> RecordHeader(
headerKey.field,
headerValue.toByteArray()
)
}
}
headers.forEach { header ->
record.headers().add(header)
}
producer.send(record)
}.launchIn(builderScope)
}
private suspend fun launchTopology(
id: String,
topology: Topology,
@ -105,7 +177,7 @@ internal class KafkatorioTopology(
streams.setStateListener { newState: KafkaStreams.State, _ ->
when (newState) {
KafkaStreams.State.NOT_RUNNING,
KafkaStreams.State.ERROR -> cont.resume(newState)
KafkaStreams.State.ERROR -> if (cont.isActive) cont.resume(newState)
else -> Unit // do nothing
}
}
@ -124,6 +196,7 @@ internal class KafkatorioTopology(
}
}
private fun printTopologyDescription(appId: String, topology: Topology) {
val description: TopologyDescription = topology.describe()
@ -143,6 +216,7 @@ internal class KafkatorioTopology(
)
}
companion object {
private fun Map<String, String>.toProperties(): Properties =
entries.fold(Properties()) { props, (k, v) ->

18
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/admin/kafkatorioTopics.kt
View file

@ -48,14 +48,15 @@ fun allTopics(): Set<String> = buildSet {
val KafkatorioPacketData.topicName: String
get() = when (this) {
is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class)
is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class)
is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class)
is PrototypesUpdate -> packetTopicNames.getValue(PrototypesUpdate::class)
is SurfaceUpdate -> packetTopicNames.getValue(SurfaceUpdate::class)
is EntityUpdate -> packetTopicNames.getValue(EntityUpdate::class)
is MapChunkUpdate -> packetTopicNames.getValue(MapChunkUpdate::class)
is PlayerUpdate -> packetTopicNames.getValue(PlayerUpdate::class)
is ConfigurationUpdate -> packetTopicNames.getValue(ConfigurationUpdate::class)
is ConsoleChatUpdate -> packetTopicNames.getValue(ConsoleChatUpdate::class)
is ConsoleCommandUpdate -> packetTopicNames.getValue(ConsoleCommandUpdate::class)
is PrototypesUpdate -> packetTopicNames.getValue(PrototypesUpdate::class)
is SurfaceUpdate -> packetTopicNames.getValue(SurfaceUpdate::class)
is EntityUpdate -> packetTopicNames.getValue(EntityUpdate::class)
is MapChunkUpdate -> packetTopicNames.getValue(MapChunkUpdate::class)
is PlayerUpdate -> packetTopicNames.getValue(PlayerUpdate::class)
is KafkatorioPacketData.Error -> packetTopicNames.getValue(KafkatorioPacketData.Error::class)
}
@ -68,6 +69,7 @@ private val packetTopicNames: Map<KClass<out KafkatorioPacketData>, String> = ma
EntityUpdate::class to "entity",
MapChunkUpdate::class to "map-chunk",
PlayerUpdate::class to "player",
KafkatorioPacketData.Error::class to "error",
).mapValues { (_, v) ->
"$DOMAIN.packet.$v"
}

3
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/config/ApplicationProperties.kt
View file

@ -12,6 +12,9 @@ data class ApplicationProperties(
val tileDir: Path,
val kafkaAdminConfig: Map<String, String>,
val kafkaStreamsConfig: Map<String, String>,
val socketServerHost: String,
val socketServerPort: Int,
) {
companion object {

10
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/eventProcessor.kt
View file

@ -3,6 +3,7 @@ package dev.adamko.kafkatorio.processor
import dev.adamko.kafkatorio.processor.admin.KafkatorioKafkaAdmin
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.tileserver.SyslogSocketServer
import dev.adamko.kafkatorio.processor.tileserver.WebMapTileServer
import dev.adamko.kafkatorio.processor.tileserver.WebsocketServer
import dev.adamko.kafkatorio.processor.tileserver.webServer
@ -11,7 +12,6 @@ import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.currentCoroutineContext
import kotlinx.coroutines.job
import kotlinx.coroutines.launch
import kotlinx.serialization.Serializable
import org.http4k.server.Http4kServer
@ -22,12 +22,18 @@ suspend fun main() {
val admin = KafkatorioKafkaAdmin(appProps)
admin.createKafkatorioTopics()
val syslogServer = SyslogSocketServer(appProps)
val wsServer = WebsocketServer()
val tileServer = WebMapTileServer(appProps)
val topology = KafkatorioTopology(appProps, wsServer, syslogServer)
coroutineScope {
launch {
val topology = KafkatorioTopology(wsServer, appProps)
syslogServer.start()
}
launch {
topology.start()
println("launched KafkatorioTopology")

14
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/misc/DebounceProcessor.kt
View file

@ -30,14 +30,17 @@ class DebounceProcessor<K, V>(
private val storeName: String,
) : Processor<K, V, K, V> {
private val checkInterval: DurationJvm = checkInterval.toJavaDuration()
private lateinit var state: State<K, V>
override fun init(context: ProcessorContext<K, V>) {
state = Active(context)
}
override fun process(record: Record<K, V>) {
(state as Active).apply {
val (key, value, _) = record
@ -49,21 +52,28 @@ class DebounceProcessor<K, V>(
}
}
override fun close() {
(state as? Active)?.schedule?.cancel()
state = Closed()
}
private sealed interface State<K, V>
private class Closed<K, V> : State<K, V>
private inner class Active(
val context: ProcessorContext<K, V>
) : State<K, V> {
val store: TimestampedKeyValueStore<K, V> = context.getStateStore(storeName)
val schedule: Cancellable =
context.schedule(checkInterval, PunctuationType.WALL_CLOCK_TIME, ::checkRecords)
val schedule: Cancellable = context.schedule(
checkInterval,
PunctuationType.WALL_CLOCK_TIME,
::checkRecords
)
private fun checkRecords(now: Long) {
store.useAll {

168
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/tileserver/SyslogSocketServer.kt Normal file
View file

@ -0,0 +1,168 @@
package dev.adamko.kafkatorio.processor.tileserver
import com.github.palindromicity.syslog.AllowableDeviations
import com.github.palindromicity.syslog.SyslogParser
import com.github.palindromicity.syslog.SyslogParserBuilder
import com.github.palindromicity.syslog.SyslogSpecification
import com.github.palindromicity.syslog.dsl.SyslogFieldKeys
import dev.adamko.kafkatorio.processor.config.ApplicationProperties
import dev.adamko.kafkatorio.processor.tileserver.SyslogMsg.Companion.toSyslogMsg
import io.ktor.network.selector.SelectorManager
import io.ktor.network.sockets.aSocket
import io.ktor.network.sockets.awaitClosed
import io.ktor.network.sockets.openReadChannel
import io.ktor.utils.io.readUTF8Line
import java.util.EnumSet
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.SharedFlow
import kotlinx.coroutines.flow.asSharedFlow
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.yield
class SyslogSocketServer(
appProps: ApplicationProperties,
dispatcher: CoroutineDispatcher = Dispatchers.IO,
) {
private val selectorManager: SelectorManager = SelectorManager(dispatcher)
private val serverSocketTcp = aSocket(selectorManager)
.tcp()
.bind(
appProps.socketServerHost,
appProps.socketServerPort,
)
// private val serverSocketUdp: BoundDatagramSocket = aSocket(selectorManager)
// .udp()
// .bind(
// localAddress = InetSocketAddress(
// appProps.socketServerHost,
// appProps.socketServerPortUdp,
// ),
// )
private val syslogParser: SyslogParser = SyslogParserBuilder()
.withDeviations(EnumSet.of(AllowableDeviations.PRIORITY, AllowableDeviations.VERSION))
.forSpecification(SyslogSpecification.RFC_5424)
.build()
private val _messages: MutableSharedFlow<SyslogMsg> = MutableSharedFlow()
val messages: SharedFlow<SyslogMsg>
get() = _messages.asSharedFlow()
suspend fun start() {
// udp()
tcp()
}
// private suspend fun udp() {
// println("[SyslogSocketServer] started listening: ${serverSocketUdp.localAddress}")
//
// serverSocketUdp.incoming
// .receiveAsFlow()
// .map {udpPacket: Datagram ->
// val msg = udpPacket.packet.readText()
// println("[SyslogSocketServer] received msg: $msg")
// syslogParser.parseLine(msg).toSyslogMsg()
// }.collect(_messages)
// }
private suspend fun tcp() = coroutineScope {
println("[SyslogSocketServer] started listening: ${serverSocketTcp.localAddress}")
while (isActive) {
val socket = serverSocketTcp.accept()
launch {
val channel = socket.openReadChannel()
try {
while (socket.socketContext.isActive) {
val line = channel.readUTF8Line()
println("[SyslogSocketServer] received: $line")
val syslog = syslogParser.parseLine(line).toSyslogMsg()
_messages.emit(syslog)
yield()
}
} catch (e: Exception) {
socket.awaitClosed()
}
}
}
yield()
}
}
data class SyslogMsg(
val src: Map<String, String?>
) {
val message: String?
get() = src[SyslogFieldKeys.MESSAGE.field]
val headerAppName: String?
get() = src[SyslogFieldKeys.HEADER_APPNAME.field]
val headerHostName: String?
get() = src[SyslogFieldKeys.HEADER_HOSTNAME.field]
val headerPri: String?
get() = src[SyslogFieldKeys.HEADER_PRI.field]
val headerPriSeverity: String?
get() = src[SyslogFieldKeys.HEADER_PRI_SEVERITY.field]
val headerPriFacility: String?
get() = src[SyslogFieldKeys.HEADER_PRI_FACILITY.field]
val headerProcId: String?
get() = src[SyslogFieldKeys.HEADER_PROCID.field]
val headerTimestamp: String?
get() = src[SyslogFieldKeys.HEADER_TIMESTAMP.field]
val headerMsgId: String?
get() = src[SyslogFieldKeys.HEADER_MSGID.field]
val headerVersion: String?
get() = src[SyslogFieldKeys.HEADER_VERSION.field]
val structuredBase: String?
get() = src[SyslogFieldKeys.STRUCTURED_BASE.field]
val structuredElementIdFmt: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_FMT.field]
val structuredElementIdPnameFmt: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_FMT.field]
val structuredElementIdPnamePattern: String?
get() = src[SyslogFieldKeys.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.field]
companion object {
fun Map<String, Any?>.toSyslogMsg(): SyslogMsg {
val map = entries.mapNotNull { (k, v) ->
when (v) {
is String -> k to v
else -> null
}
}.toMap()
return SyslogMsg(map)
}
}
}

33
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/colourMapChunks.kt
View file

@ -17,6 +17,7 @@ import dev.adamko.kotka.extensions.groupedAs
import dev.adamko.kotka.extensions.materializedAs
import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.repartitionedAs
import dev.adamko.kotka.extensions.stream
import dev.adamko.kotka.extensions.streams.filter
import dev.adamko.kotka.extensions.streams.map
import dev.adamko.kotka.extensions.streams.mapValues
@ -55,26 +56,30 @@ fun colourMapChunks(builder: StreamsBuilder): Topology {
chunkTilesColouredTable032.streamMapChunkColouredTo(
ChunkSize.CHUNK_032,
TOPIC_MAP_CHUNK_COLOURED_032_STATE
TOPIC_MAP_CHUNK_COLOURED_032_STATE,
)
val chunkTilesColoured032Debounced: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
val chunkTilesColoured: KStream<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
builder.stream(
TOPIC_MAP_CHUNK_COLOURED_STATE,
consumedAs(
"$pid.consume.map-chunks-coloured",
kxsBinary.serde(),
kxsBinary.serde(),
"$pid.consume.map-chunks-debounced",
kxsBinary.serde(ServerMapChunkId.serializer()),
kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
),
TOPIC_MAP_CHUNK_COLOURED_STATE,
)
mapOf(
ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
// ChunkSize.CHUNK_032 to ChunkSize.CHUNK_064,
// ChunkSize.CHUNK_064 to ChunkSize.CHUNK_128,
// ChunkSize.CHUNK_128 to ChunkSize.CHUNK_256,
// ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
ChunkSize.CHUNK_032 to ChunkSize.CHUNK_256,
ChunkSize.CHUNK_256 to ChunkSize.CHUNK_512,
).forEach { (from, to) ->
chunkTilesColoured032Debounced
chunkTilesColoured
.filter("$pid.combine-chunks.filter-$from-to-$to") { key, _ ->
key.chunkSize == from
}.groupByChunkPosition(to)
@ -86,9 +91,9 @@ fun colourMapChunks(builder: StreamsBuilder): Topology {
namePrefix = "groupTilesMapChunks",
sourceTopic = TOPIC_MAP_CHUNK_COLOURED_032_STATE,
sinkTopic = TOPIC_MAP_CHUNK_COLOURED_STATE,
inactivityDuration = 15.seconds,
keySerde = kxsBinary.serde<ServerMapChunkId>(),
valueSerde = kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
inactivityDuration = 5.seconds,
keySerde = kxsBinary.serde(ServerMapChunkId.serializer()),
valueSerde = kxsBinary.serde(ServerMapChunkTiles.serializer(ColourHex.serializer())),
)
}
@ -182,7 +187,7 @@ private fun KTable<ServerMapChunkId, ServerMapChunkTiles<TileProtoHashCode>>.enr
val missingProtos = mutableSetOf<TileProtoHashCode>()
val tileColours = tiles.map.mapValues { (_, code) ->
colourDict.map.getOrElse(code) {
colourDict.getOrElse(code) {
missingProtos.add(code)
ColourHex.TRANSPARENT
}

22
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/factorioServerPackets.kt
View file

@ -5,7 +5,9 @@ import dev.adamko.kafkatorio.processor.admin.TOPIC_SRC_SERVER_LOG
import dev.adamko.kafkatorio.processor.admin.topicName
import dev.adamko.kafkatorio.processor.serdes.jsonMapper
import dev.adamko.kafkatorio.schema.common.FactorioServerId
import dev.adamko.kafkatorio.schema.common.Tick
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
import dev.adamko.kotka.extensions.component1
import dev.adamko.kotka.extensions.component2
import dev.adamko.kotka.extensions.consumedAs
@ -31,8 +33,26 @@ fun factorioServerPacketStream(
Serdes.String(),
)
).map("decode-packets") { serverId: String, value: String ->
val packet = jsonMapper.decodeFromString<KafkatorioPacket>(value)
val packet = runCatching {
jsonMapper.decodeFromString<KafkatorioPacket>(value)
}.getOrElse { e ->
KafkatorioPacket(
modVersion = "unknown",
tick = Tick(0u),
data = KafkatorioPacketData.Error(
message = e.message,
rawValue = value,
)
)
}
if (packet.data is KafkatorioPacketData.Error) {
println("error parsing $TOPIC_SRC_SERVER_LOG message: ${packet.data}")
}
FactorioServerId(serverId) to packet
}.filter { _, value ->
value.data !is KafkatorioPacketData.Error
}
}

1
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt
View file

@ -1 +0,0 @@
package dev.adamko.kafkatorio.processor.topology

8
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/prototypes.kt
View file

@ -29,7 +29,9 @@ value class TileProtoHashCode private constructor(val code: Int) {
@JvmInline
@Serializable
value class TileColourDict(val map: Map<TileProtoHashCode, ColourHex>)
value class TileColourDict(
private val map: Map<TileProtoHashCode, ColourHex>
) : Map<TileProtoHashCode, ColourHex> by map
/** Get the latest map-tile colours per server. */
@ -48,9 +50,9 @@ fun KStream<FactorioServerId, PrototypesUpdate>.createTilePrototypeTable()
TileColourDict(map)
}.filter("$pid.filterMapTileProtos") { _, dict: TileColourDict ->
dict.map.isNotEmpty()
dict.isNotEmpty()
}.peek { serverId, dict: TileColourDict ->
println("$pid: server $serverId has TileColourDict[${dict.map.size}]: ${dict.map.entries.joinToString()}")
println("$pid: server $serverId has TileColourDict[${dict.size}]: ${dict.entries.joinToString()}")
}.repartition(
repartitionedAs(
"$pid.pre-table-repartition",

0
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/playerUpdates.kt → modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/websocketBroadcast.kt
View file

8
modules/events-server/src/main/resources/config.yml
View file

@ -1,12 +1,18 @@
# Factorio Events Processor config
web-port: ${SERVER_PORT:-12080}
web-port: ${SERVER_PORT:-8080}
tile-dir: ${TILE_DIR:-./tile-dir}
socket-server-host: ${SOCKET_SERVER_HOST:-127.0.0.1}
socket-server-port: ${SOCKET_SERVER_PORT:-9002}
kafka-admin-config:
client.id: kafkatorio.events-server-admin
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}
kafka-streams-config:
application.id: kafkatorio-events-processor
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:-http://localhost:9092}

26
modules/infra-factorio-client/build.gradle.kts
View file

@ -1,3 +1,4 @@
import dev.adamko.kafkatorio.factoriomod.FactorioMod
import dev.adamko.kafkatorio.gradle.asConsumer
import dev.adamko.kafkatorio.gradle.factorioModAttributes
import dev.adamko.kafkatorio.gradle.not
@ -34,10 +35,11 @@ dependencies {
factorioMod(projects.modules.eventsMod)
}
//<editor-fold desc="Mod deployment tasks">
val deployModToClient by tasks.registering(Copy::class) {
val deployModToLocalClient by tasks.registering(Copy::class) {
description = "Copy the mod to the Factorio client"
group = project.name
group = FactorioMod.TASK_GROUP
onlyIf { clientModsDirectory.orNull?.asFile?.exists() == true }
@ -50,19 +52,20 @@ val deployModToClient by tasks.registering(Copy::class) {
}
//</editor-fold>
//<editor-fold desc="Factorio client lifecycle tasks">
fun ExecOperations.isFactorioRunning(): Spec<Task> = ProcessRunningSpec(this, "factorio.exe")
val clientLaunch by tasks.registering(Exec::class) {
description = "Run Factorio Steam game client"
group = project.name
description = "Run local Factorio Steam game client"
group = FactorioMod.TASK_GROUP
onlyIf(!serviceOf<ExecOperations>().isFactorioRunning())
onlyIf { steamExe.orNull?.asFile?.exists() == true }
onlyIf { clientModsDirectory.orNull?.asFile?.exists() == true }
dependsOn(deployModToClient)
dependsOn(deployModToLocalClient)
mustRunAfter(clientKill, ":modules:infra-factorio-server:processRun")
commandLine = parseSpaceSeparatedArgs(
@ -74,8 +77,8 @@ val clientLaunch by tasks.registering(Exec::class) {
}
val clientKill by tasks.registering(Exec::class) {
description = "Run Factorio Steam game client"
group = project.name
description = "Stop the local Factorio Steam game client"
group = FactorioMod.TASK_GROUP
onlyIf(serviceOf<ExecOperations>().isFactorioRunning())
@ -84,11 +87,16 @@ val clientKill by tasks.registering(Exec::class) {
}
//</editor-fold>
tasks.build { dependsOn(deployModToClient) }
tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) {
group = FactorioMod.TASK_GROUP
dependsOn(deployModToLocalClient)
}
tasks.processRun {
dependsOn(
deployModToClient,
deployModToLocalClient,
clientLaunch,
)
}

20
modules/infra-factorio-server/build.gradle.kts
View file

@ -1,3 +1,4 @@
import dev.adamko.kafkatorio.factoriomod.FactorioMod
import dev.adamko.kafkatorio.gradle.asConsumer
import dev.adamko.kafkatorio.gradle.factorioModAttributes
import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
@ -6,21 +7,25 @@ plugins {
dev.adamko.kafkatorio.infra.`docker-compose`
}
val dockerSrcDir: Directory by extra
val factorioServerDataDir: Directory = dockerSrcDir.dir("factorio-server")
val factorioMod: Configuration by configurations.creating {
asConsumer()
factorioModAttributes(objects)
}
dependencies {
factorioMod(projects.modules.eventsMod)
}
val deployModToServer by tasks.registering(Copy::class) {
val deployModToLocalServer by tasks.registering(Copy::class) {
description = "Copy the mod to the Factorio Docker server"
group = project.name
group = FactorioMod.TASK_GROUP
dependsOn(factorioMod)
@ -34,22 +39,29 @@ val deployModToServer by tasks.registering(Copy::class) {
}
}
tasks.dockerDown {
commandLine = parseSpaceSeparatedArgs(""" docker-compose stop """)
}
tasks.dockerUp {
dependsOn(
deployModToServer,
deployModToLocalServer,
":modules:infra-kafka-cluster:processRun",
)
}
tasks.dockerEnv {
properties("FACTORIO_VERSION" to libs.versions.factorio.get())
}
tasks.build { dependsOn(deployModToServer) }
tasks.register(FactorioMod.PUBLISH_MOD_LOCAL_TASK_NAME) {
group = FactorioMod.TASK_GROUP
dependsOn(deployModToLocalServer)
}
idea {

2
modules/infra-factorio-server/src/.env
View file

@ -1,4 +1,4 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
FACTORIO_VERSION=1.1.59
KAFKATORIO_VERSION=0.6.0
KAFKATORIO_VERSION=0.6.1

13
modules/infra-factorio-server/src/docker-compose.yml
View file

@ -4,8 +4,6 @@ services:
factorio-server:
image: factoriotools/factorio:${FACTORIO_VERSION}
# logging:
# driver: "json-file"
container_name: "factorio-server"
restart: "no"
networks:
@ -18,12 +16,11 @@ services:
- "27015:27015/tcp" # rcon
volumes:
- "./factorio-server:/factorio"
# logging:
# driver: "mickyg/kafka-logdriver:latest"
# options:
# KAFKA_BROKER_ADDR: "kafka-kraft"
# LOG_TOPIC: "dockerlogs-factorio-server"
# LOG_TAG: "factorio-server"
logging:
driver: syslog
options:
syslog-address: "tcp://host.docker.internal:9002"
syslog-format: "rfc5424"
# environment:
# RCON_PORT: 27715

2
modules/infra-kafka-cluster/src/.env
View file

@ -1,7 +1,7 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
GRAFANA_VERSION=8.5.0
KAFKATORIO_VERSION=0.6.0
KAFKATORIO_VERSION=0.6.1
KAFKA_DL_URL=https\://dlcdn.apache.org/kafka/3.2.0/kafka_2.13-3.2.0.tgz
KAFKA_VERSION=3.2.0
PROMETHEUS_VERSION=v2.34.0

18
modules/infra-kafka-cluster/src/docker-compose.yml
View file

@ -36,8 +36,8 @@ services:
- "traefik-nw"
labels:
- "traefik.enable=true"
# ports:
# - "8080:8080"
ports:
- "18090:8080"
environment:
KAFKA_CLUSTERS_0_NAME: "${COMPOSE_PROJECT_NAME}"
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka:29092"
@ -46,20 +46,6 @@ services:
KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: "http://connect:8083"
# - KAFKA_CLUSTERS_0_ZOOKEEPER=kafka:9093
# kafka-connect:
# image: dev-adamko/kafka-connect
# container_name: "kafka-connect"
# build:
# dockerfile: "kafka.Dockerfile"
# target: "kafka-connect"
# args:
# KAFKA_DL_URL: "https://dlcdn.apache.org/kafka/3.0.0/kafka_2.13-3.0.0.tgz"
# depends_on:
# - kafka-kraft
# ports:
# - "8083:8083"
#
# grafana:
# image: "grafana/grafana:${GRAFANA_VERSION}"
# container_name: "grafana"

19
modules/infra-kafka-cluster/src/kafka.Dockerfile
View file

@ -40,22 +40,3 @@ COPY --chmod=755 ./entrypoint.sh ./entrypoint.sh
# launch the broker in KRaft mode, which means that it runs without ZooKeeper
ENTRYPOINT ["/kafka/entrypoint.sh"]
### Kafka Connect ##
#FROM openjdk:11 as kafka-connect
## https://github.com/confluentinc/cp-docker-images/blob/5.3.3-post/debian/kafka-connect-base/Dockerfile
#
#ENV COMPONENT=kafka-connect
#
#WORKDIR /kafka-connect
#
#COPY --from=Kafka-Download /kafka .
#
## Default kafka-connect rest.port
#EXPOSE 8083
#
#VOLUME ["/etc/${COMPONENT}/jars", "/etc/${COMPONENT}/secrets"]
#
## launch the broker in KRaft mode, which means that it runs without ZooKeeper
#ENTRYPOINT ["./bin/connect-standalone.sh", "./config/connect-standalone.properties"]

116
modules/infra-kafka-connect/build.gradle.kts
View file

@ -1,116 +0,0 @@
import dev.adamko.kafkatorio.gradle.asConsumer
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
plugins {
dev.adamko.kafkatorio.infra.`docker-compose`
dev.adamko.kafkatorio.lang.`kotlin-jvm`
}
val dockerSrcDir: Directory by extra(layout.projectDirectory.dir("src/main/docker"))
val connectorJarDir: Directory = dockerSrcDir.dir("camel-kafka-connectors")
val camelConnectorDependencies: Configuration by configurations.creating { asConsumer() }
fun DependencyHandlerScope.kafkaConnector(
name: String,
group: String = "org.apache.camel.kafkaconnector",
version: String = libs.versions.camel.kafkaConnectors.get(),
classifier: String = "package",
extension: String = "tar.gz",
) {
camelConnectorDependencies(group, name, version, ext = extension, classifier = classifier)
}
dependencies {
implementation(platform(projects.modules.versionsPlatform))
kotlin("reflect")
kafkaConnector("camel-ahc-ws-kafka-connector")
kafkaConnector("camel-websocket-kafka-connector")
implementation("com.github.adamko-dev:json5-kotlin:2.0.3")
implementation(libs.bundles.http4k)
implementation(libs.kotlinx.serialization.core)
implementation(libs.kotlinx.serialization.json)
}
// disabled for now...
val projectEnabled = false
val dockerBuildKafkaConnect by tasks.registering(Exec::class) {
enabled = projectEnabled
group = "docker-compose"
dependsOn(downloadCamelConnectors, tasks.dockerEnv)
logging.captureStandardOutput(LogLevel.LIFECYCLE)
inputs.dir(dockerSrcDir)
workingDir = dockerSrcDir.asFile
commandLine = parseSpaceSeparatedArgs(""" docker-compose build connect """)
}
//tasks.assemble { dependsOn(dockerBuildKafkaConnect) }
val downloadCamelConnectors by tasks.registering(Sync::class) {
enabled = projectEnabled
group = project.name
description = "Retrieve Camel Kafka Connectors from Maven, in preparation for the Docker build"
dependsOn(camelConnectorDependencies)
from(
provider { camelConnectorDependencies }
.map { eventsSchema ->
eventsSchema.incoming
.artifacts
.artifactFiles
}
)
into(temporaryDir)
doLast {
sync {
fileTree(temporaryDir)
.asFileTree
.forEach { zip ->
from(tarTree(zip))
}
into(connectorJarDir)
}
}
}
tasks.dockerUp {
enabled = projectEnabled
dependsOn(":modules:infra-kafka-cluster:dockerUp")
dependsOn(downloadCamelConnectors)
}
tasks.dockerEnv {
enabled = projectEnabled
properties(
"CONNECTOR_ID" to "kafkatorio-connect",
"KAFKA_CONNECT_VERSION" to libs.versions.confluent.kafkaConnect.get(),
)
}
idea {
module {
excludeDirs.add(file("src/main/docker/camel-kafka-connectors"))
}
}
tasks.withType<KotlinCompile>().configureEach {
kotlinOptions.freeCompilerArgs += listOf(
"-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}

4
modules/infra-kafka-connect/src/main/docker/.env
View file

@ -1,4 +0,0 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
CONNECTOR_ID=kafkatorio-connect
KAFKA_CONNECT_VERSION=7.0.1

1
modules/infra-kafka-connect/src/main/docker/.gitignore vendored
View file

@ -1 +0,0 @@
camel-kafka-connectors/

55
modules/infra-kafka-connect/src/main/docker/docker-compose.yml
View file

@ -1,55 +0,0 @@
version: "3.9"
services:
connect:
image: adamko-dev/kafkatorio-kafka-connect:${KAFKA_CONNECT_VERSION}
hostname: ${CONNECTOR_ID}
container_name: ${CONNECTOR_ID}
build:
context: "."
dockerfile: kafka-connect.Dockerfile
args:
KAFKA_CONNECT_VERSION: ${KAFKA_CONNECT_VERSION}
# FILE_PULSE_VERSION: ${CONNECTOR_FILE_PULSE_VERSION}
# CAMEL_KC_VERSION: ${CONNECTOR_CAMEL_KAFKA_VERSION}
# CAMEL_CONNECTOR: ${CAMEL_FILE_CONNECTOR}
ports:
- "8083:8083"
- "8095:8095" # ws-async (client?)
- "9292:9292" # websocket server(?)
# volumes:
# - "./connect/input:/warehouse/input"
# - "./connect/components:/usr/share/confluent-hub-components"
# volumes:
# - "./camel-kafka-connectors:/usr/share/java"
networks:
- "kafkatorio-nw"
environment:
CONNECT_BOOTSTRAP_SERVERS: "kafka:29092"
# CONNECT_REST_ADVERTISED_HOST_NAME: ${CONNECTOR_ID}
CONNECT_REST_ADVERTISED_HOST_NAME: connect
# CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://${SCHEMA_REGISTRY}:8081"
# CONNECT_REST_PORT: 28083
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: ${CONNECTOR_ID}
CONNECT_CONFIG_STORAGE_TOPIC: ${CONNECTOR_ID}-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: ${CONNECTOR_ID}-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: ${CONNECTOR_ID}-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter"
# CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/camel-kafka-connectors"
CONNECT_LOG4J_ROOT_LOGLEVEL: INFO
CONNECT_LOG4J_LOGGERS: "org.apache.zookeeper=INFO,org.I0Itec.zkclient=ERROR,org.reflections=ERROR"
# CONNECT_LISTENERS: "http://localhost:28083"
networks:
kafkatorio-nw:
name: "kafkatorio-nw"
external: true

19
modules/infra-kafka-connect/src/main/docker/kafka-connect.Dockerfile
View file

@ -1,19 +0,0 @@
ARG KAFKA_CONNECT_VERSION
FROM confluentinc/cp-kafka-connect-base:${KAFKA_CONNECT_VERSION}
ENV CONNECT_VERSION=${KAFKA_CONNECT_VERSION}
USER 0:0
COPY ./camel-kafka-connectors /usr/share/camel-kafka-connectors
#WORKDIR /usr/share/java/
#
##ENV CAMEL_KC_VERSION=0.9.0
##ENV CAMEL_CONNECTOR=camel-file-kafka-connector
#
#ENV URL=https://repo.maven.apache.org/maven2/org/apache/camel/kafkaconnector/camel-file-kafka-connector/0.9.0/camel-file-kafka-connector-0.9.0-package.tar.gz
#
#RUN curl -L $URL | tar -xzvf - --strip=0 $CAMEL_CONNECTOR

10
modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/Resources.kt
View file

@ -1,10 +0,0 @@
package dev.adamko.kafkatorio.kafkaconnect
import java.io.File
object Resources {
fun loadResource(name: String): File =
File(requireNotNull(Resources::class.java.getResource(name)?.file) {
"could not load resource '$name'"
})
}

92
modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt
View file

@ -1,92 +0,0 @@
package dev.adamko.kafkatorio.kafkaconnect
import at.syntaxerror.json5.Json5Module
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.jsonPrimitive
import org.http4k.client.OkHttp
import org.http4k.core.Method.GET
import org.http4k.core.Method.PUT
import org.http4k.core.Request
import org.http4k.core.Uri
val jsonMapper = Json {
prettyPrint = true
prettyPrintIndent = " "
}
val j5 = Json5Module()
fun main() {
val connectorHostUrl = Uri.of("http://localhost:8083")
val configFile = Resources.loadResource("/configs/ckc-websocket-lua-objects.json5")
val jsonObject: JsonObject = j5.decodeObject(configFile.readText())
println("json5 connector config: $jsonObject")
val configJson = jsonMapper.encodeToString(jsonObject)
println("json connector config: $configJson")
val connectorName = jsonObject["name"]!!.jsonPrimitive.content
val request: Request =
Request(PUT, connectorHostUrl.path("/connectors/$connectorName/config"))
.header("Content-Type", "application/json")
.header("Accept", "application/json")
// .header("Accept-Encoding", "identity")
.body(configJson)
val client = OkHttp()
// check the status of KC
val statusResponse = client(Request(GET, connectorHostUrl))
println(statusResponse.toMessage())
val response = client(request)
println(response.toMessage())
println(
jsonMapper.encodeToString(
jsonMapper.decodeFromString<JsonElement>(response.body.toString())
)
)
}
//val updateConnectors by tasks.registering {
// group = project.name
//
// val configFile = layout.projectDirectory.file("configs/websocket-async-lua-objects.json5")
// inputs.file(configFile)
//
// doLast {
// val fileText = resources.text.fromFile(configFile)
//
// val jsonObject: JsonObject = j5.decodeObject(fileText.asString())
//
// logger.lifecycle("out: $jsonObject")
// val configJson = jsonMapper.encodeToString(jsonObject)
// logger.lifecycle("configJson: $configJson")
//
// val request = Request(
// PUT,
// Uri.of("http://localhost:8083").path("/connectors/kafkatorio-ws-connector/config")
// ).header("Content-Type", "application/json")
// .body(configJson)
//
//
// val client = OkHttp()
// val response = client(request)
//
// println(response.toMessage())
// println(
// jsonMapper.encodeToString(
// jsonMapper.decodeFromString<JsonElement>(response.body.toString())
// )
// )
// }
//
//}

41
modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt
View file

@ -1,41 +0,0 @@
package dev.adamko.kafkatorio.kafkaconnect
import java.time.Duration
import org.http4k.client.WebsocketClient
import org.http4k.core.Uri
import org.http4k.websocket.WsMessage
fun main() {
val client = WebsocketClient.blocking(
uri = Uri.of("ws://localhost:9073/ws"),
// autoReconnection = true,
// timeout = Duration.ofSeconds(30)
)
// client.send(WsMessage("server sent on connection"))
client.received()
.take(100)
.forEachIndexed { index, wsMessage ->
println("[$index] ${wsMessage.bodyString()}")
}
// val nonBlockingClient = WebsocketClient.nonBlocking(
// uri = Uri.of("ws://localhost:9097/lua-objects"),
// onError = { it.printStackTrace() },
// onConnect = { ws ->
// println("onConnect, ws.upgradeRequest ${ws.upgradeRequest}")
//// ws.send(WsMessage("client sent on connection"))
// }
// )
//
// nonBlockingClient.onMessage { msg -> print(msg.bodyString()) }
// nonBlockingClient.onClose {
// println("non-blocking client closing")
// }
//// nonBlockingClient.onError { it.printStackTrace() }
//
// Thread.sleep(Duration.ofMinutes(1).toMillis())
}

130
modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5
View file

@ -1,130 +0,0 @@
{
"name": "ckc-websocket-lua-objects",
"connector.class": "org.apache.camel.kafkaconnector.websocket.CamelWebsocketSinkConnector",
"topics.regex": "^Lua.*$",
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"value.converter": "org.apache.kafka.connect.storage.StringConverter",
// The hostname. The default value is 0.0.0.0. Setting this option on the component will use the
// component configured value as default.
"camel.sink.path.host": "0.0.0.0",
// "camel.sink.path.host": "host.docker.internal",
// The port number. The default value is 9292. Setting this option on the component will use the
// component configured value as default.
"camel.sink.path.port": "9292",
// Name of the websocket channel to use
"camel.sink.path.resourceUri": "lua-objects",
"camel.sink.endpoint.sendToAll": "true",
"camel.component.websocket.host":"0.0.0.0",
"camel.component.websocket.port": "8084",
// Can be used to set the size in bytes that the websocket created by the websocketServlet may
// be accept before closing. (Default is -1 - or unlimited)
// "camel.sink.endpoint.maxBinaryMessageSize": -1,
// Whether the producer should be started lazy (on the first message). By starting lazy you can
// use this to allow CamelContext and routes to startup in situations where a producer may otherwise
// fail during starting and cause the route to fail being started. By deferring this startup to be
// lazy then the startup failure can be handled during routing messages via Camel's routing error handlers.
// Beware that when the first message is processed then creating and starting the producer may take a little
// time and prolong the total processing time of the processing.
// "camel.sink.endpoint.lazyStartProducer": false,
// Timeout in millis when sending to a websocket channel. The default timeout is 30000 (30 seconds).
// "camel.sink.endpoint.sendTimeout": 30000,
// To send to all websocket subscribers. Can be used to configure on endpoint level, instead of
// having to use the WebsocketConstants.SEND_TO_ALL header on the message.
// "camel.sink.endpoint.sendToAll": ,
// Set the buffer size of the websocketServlet, which is also the max frame byte size (default
// 8192)
// "camel.sink.endpoint.bufferSize": 8192,
// Set the time in ms that the websocket created by the websocketServlet may be idle before
// closing. (default is 300000)
// "camel.sink.endpoint.maxIdleTime": 300000,
// Can be used to set the size in characters that the websocket created by the websocketServlet
// may be accept before closing.
// "camel.sink.endpoint.maxTextMessageSize": ,
// Can be used to set the minimum protocol version accepted for the websocketServlet.
// (Default 13 - the RFC6455 version)
// "camel.sink.endpoint.minVersion": 13,
// The CORS allowed origins. Use to allow all.
// "camel.sink.endpoint.allowedOrigins": ,
// Whether to enable CORS
// "camel.sink.endpoint.crossOriginFilterOn": false,
// Context path for filtering CORS
// "camel.sink.endpoint.filterPath": ,
// If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX
// support for more details.
// "camel.sink.endpoint.enableJmx": false,
// To configure security using SSLContextParameters
// "camel.sink.endpoint.sslContextParameters": ,
// The hostname. The default value is 0.0.0.0
// "camel.component.websocket.host": "0.0.0.0",
// The port number. The default value is 9292
// "camel.component.websocket.port": 9292,
// Whether the producer should be started lazy (on the first message). By starting lazy you can
// use this to allow CamelContext and routes to startup in situations where a producer may otherwise
// fail during starting and cause the route to fail being started. By deferring this startup to be
// lazy then the startup failure can be handled during routing messages via Camel's routing error handlers.
// Beware that when the first message is processed then creating and starting the producer may take a little
// time and prolong the total processing time of the processing.
// "camel.component.websocket.lazyStartProducer": false,
// Whether autowiring is enabled. This is used for automatic autowiring options (the option must
// be marked as autowired) by looking up in the registry to find if there is a single instance
// of matching type, which then gets configured on the component. This can be used for automatic
// configuring JDBC data sources, JMS connection factories, AWS Clients, etc.
// "camel.component.websocket.autowiredEnabled": true,
// If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX
// support for more details.
// "camel.component.websocket.enableJmx": false,
// To set a value for maximum number of threads in server thread pool. MaxThreads\/minThreads
// or threadPool fields are required due to switch to Jetty9. The default values for maxThreads
// is 1 2 noCores.
// "camel.component.websocket.maxThreads": ,
// To set a value for minimum number of threads in server thread pool. MaxThreads\/minThreads
// or threadPool fields are required due to switch to Jetty9. The default values for minThreads
// is 1.
// "camel.component.websocket.minThreads": ,
// This is a comma-separated list of subprotocols that are supported by the application. The list
// is in priority order. The first subprotocol on this list that is proposed by the client is
// the one that will be accepted. If no subprotocol on this list is proposed by the client, then the
// websocket connection is refused. The special value 'any' means that any subprotocol is acceptable. 'any'
// can be used on its own, or as a failsafe at the end of a list of more specific protocols. 'any' will also
// match the case where no subprotocol is proposed by the client.
// "camel.component.websocket.subprotocol": "any",
// To use a custom thread pool for the server. MaxThreads\/minThreads or threadPool fields are
// required due to switch to Jetty9.
// "camel.component.websocket.threadPool": ,
// To configure security using SSLContextParameters
// "camel.component.websocket.sslContextParameters": ,
// The password for the keystore when using SSL.
// "camel.component.websocket.sslKeyPassword": ,
// The path to the keystore.
// "camel.component.websocket.sslKeystore": ,
// The password when using SSL.
// "camel.component.websocket.sslPassword": ,
// Enable usage of global SSL context parameters.
// "camel.component.websocket.useGlobalSslContextParameters": false,
}

84
modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5
View file

@ -1,84 +0,0 @@
//Exchange data with external Websocket servers using Async Http Client
{
name: "websocket-async-lua-objects",
"connector.class": "org.apache.camel.kafkaconnector.ahcws.CamelAhcwsSinkConnector",
"tasks.max": 1,
//use the kafka converters that better suit your needs, these are just defaults:
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
"value.converter": "org.apache.kafka.connect.storage.StringConverter",
//https://docs.confluent.io/platform/current/installation/configuration/connect/sink-connect-configs.html#sinkconnectorconfigs_topics
//List of topics to consume, separated by commas
// topics=
"topics.regex": "^Lua.*$",
// mandatory properties (for a complete properties list see the connector documentation):
// The URI to use such as http://hostname:port/path
"camel.sink.path.httpUri": "host.docker.internal:8095/ws/lua",
// If the option is true, then the Exchange.HTTP_URI header is ignored, and use the endpoint's
// URI for request. You may also set the throwExceptionOnFailure to be false to let the
// AhcProducer send all the fault response back.
// camel.sink.endpoint.bridgeEndpoint=
// The initial in-memory buffer size used when transferring data between Camel and AHC Client.
// camel.sink.endpoint.bufferSize=
// To use a custom HeaderFilterStrategy to filter header to and from Camel message.
// camel.sink.endpoint.headerFilterStrategy=
// Option to disable throwing the AhcOperationFailedException in case of failed responses from
// the remote server. This allows you to get all responses regardless of the HTTP status code.
// camel.sink.endpoint.throwExceptionOnFailure=
// If enabled and an Exchange failed processing on the consumer side, and if the caused
// Exception was send back serialized in the response as a application\/x-java-serialized-object
// content type (for example using Jetty or Servlet Camel components). On the producer side the
// exception will be deserialized and thrown as is, instead of the AhcOperationFailedException.
// The caused exception is required to be serialized. This is by default turned off.
// If you enable this then be aware that Java will deserialize the incoming data from the
// request to Java and that can be a potential security risk.
// camel.sink.endpoint.transferException=
// Define if the Connection Close header has to be added to HTTP Request. This parameter is false by default
// camel.sink.endpoint.connectionClose=
// Configure a cookie handler to maintain a HTTP session
// camel.sink.endpoint.cookieHandler=
// Whether the producer should be started lazy (on the first message). By starting lazy you can
// use this to allow CamelContext and routes to startup in situations where a producer
// may otherwise fail during starting and cause the route to fail being started. By deferring
// this startup to be lazy then the startup failure can be handled during routing messages via
// Camel's routing error handlers. Beware that when the first message is processed then creating
// and starting the producer may take a little time and prolong the total processing time of the
// processing.
// camel.sink.endpoint.lazyStartProducer=
// To enable streaming to send data as multiple text fragments.
// camel.sink.endpoint.useStreaming=
// To use a custom AhcBinding which allows to control how to bind between AHC and Camel.
// camel.sink.endpoint.binding=
// To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance.
// camel.sink.endpoint.clientConfig=
// To configure the AsyncHttpClientConfig using the key\/values from the Map.
// camel.sink.endpoint.clientConfigOptions=
// To configure the AsyncHttpClientConfig Realm using the key\/values from the Map.
// camel.sink.endpoint.clientConfigRealmOptions=
// Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. This reference overrides any configured SSLContextParameters at the component level. See Using the JSSE Configuration Utility. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level.
// camel.sink.endpoint.sslContextParameters=
// Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail during starting and cause the route to fail being started. By deferring this startup to be lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. Beware that when the first message is processed then creating and starting the producer may take a little time and prolong the total processing time of the processing.
// camel.component.ahc-ws.lazyStartProducer=
// Whether to allow java serialization when a request uses context-type=application\/x-java-serialized-object This is by default turned off. If you enable this then be aware that Java will deserialize the incoming data from the request to Java and that can be a potential security risk.
// camel.component.ahc-ws.allowJavaSerializedObject=
// Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching type, which then gets configured on the component. This can be used for automatic configuring JDBC data sources, JMS connection factories, AWS Clients, etc.
// camel.component.ahc-ws.autowiredEnabled=
// To use a custom AhcBinding which allows to control how to bind between AHC and Camel.
// camel.component.ahc-ws.binding=
// To use a custom AsyncHttpClient
// camel.component.ahc-ws.client=
// To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance.
// camel.component.ahc-ws.clientConfig=
// To use a custom org.apache.camel.spi.HeaderFilterStrategy to filter header to and from Camel message.
// camel.component.ahc-ws.headerFilterStrategy=
// Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level.
// camel.component.ahc-ws.sslContextParameters=
// Enable usage of global SSL context parameters.
// camel.component.ahc-ws.useGlobalSslContextParameters=
}

2
modules/infra-kafka-pipe/src/.env
View file

@ -1,3 +1,3 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
KAFKATORIO_VERSION=0.6.0
KAFKATORIO_VERSION=0.6.1

4
modules/infra-kafka-pipe/src/docker-compose.yml
View file

@ -20,10 +20,10 @@ services:
KAFKA_HOST: "kafka:29092"
volumes:
# Windows (requires privileged/root)
# - "//var/run/docker.sock:/var/run/docker.sock"
- "//var/run/docker.sock:/var/run/docker.sock"
# Rootless:
- "/run/user/1000/docker.sock:/var/run/docker.sock"
# - "/run/user/1000/docker.sock:/var/run/docker.sock"
# Unix?
# - "/var/run/docker.sock:/var/run/docker.sock"

83
modules/web-map/build.gradle.kts
View file

@ -1,4 +1,3 @@
import dev.adamko.kafkatorio.gradle.taskProvider
import org.jetbrains.kotlin.gradle.plugin.KotlinDependencyHandler
import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackConfig
@ -17,6 +16,7 @@ kvision {
enableHiddenKotlinJsStore.set(false)
}
rootProject.extensions.configure<NodeJsRootExtension> {
// https://github.com/rjaros/kvision/issues/410
versions.webpackCli.version = "4.10.0"
@ -48,13 +48,9 @@ afterEvaluate {
kotlin {
js(IR) {
// browser()
// binaries.executable()
// }
//
// js("frontend", IR) {
// nodejs()
browser {
binaries.executable()
runTask {
outputFileName = "main.bundle.js"
sourceMaps = false
@ -87,16 +83,11 @@ kotlin {
}
}
}
binaries.executable()
// compilations["main"].packageJson {
// this.devDependencies["webpack-dev-server"] = "4.9.1"
// }
}
// jvm("backend") {
//
// }
sourceSets {
all {
@ -106,7 +97,6 @@ kotlin {
}
}
// val commonMain by getting {
main {
dependencies {
implementation(dependencies.platform(libs.kotlinx.coroutines.bom))
@ -117,13 +107,6 @@ kotlin {
implementation(projects.modules.eventsSchema)
implementation(libs.kotlinx.html)
// }
// }
// val jsMain by getting {
// dependsOn(commonMain)
//
// dependencies {
implementation(projects.modules.eventsSchema)
@ -154,37 +137,15 @@ kotlin {
// implementation(dependencies.platform(devNpm("webpack-dev-server", "4.9.1")))
}
// val webDir = file("src/main/web")
resources.srcDir("src/main/web")
// resources.srcDir("src/main/resources")
// kotlin.srcDir("src/main/kotlin")
}
// val frontendMain by getting {
// dependsOn(jsMain)
//// resources.srcDir("src/main/web")
//// resources.srcDir("src/main/resources")
// }
//
// val jsTest by getting {
// dependencies {
// implementation(kotlin("test-js"))
//
// kvision("kvision-testutils")
//
//// implementation(npm("karma", "^6.3.16"))
// }
//
// kotlin.srcDir("src/test/kotlin")
// }
test {
dependencies {
implementation(kotlin("test-js"))
kvision("kvision-testutils")
// implementation(npm("karma", "^6.3.16"))
}
}
@ -221,22 +182,22 @@ fun KotlinDependencyHandler.kvision(
//evaluationDependsOn(projects.modules.eventsSchema.dependencyProject.path)
// stop warnings when building
tasks.configureEach {
when (name) {
"productionExecutableCompileSync" ->
dependsOn(taskProvider("compileProductionExecutableKotlinJs"))
"testTestDevelopmentExecutableCompileSync" ->
dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinJs"))
// "jsTestTestDevelopmentExecutableCompileSync" ->
// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinFrontend"))
// "jsProductionExecutableCompileSync" ->
// dependsOn(taskProvider("compileProductionExecutableKotlinFrontend"))
"browserDevelopmentRun" ->
dependsOn(taskProvider("browserDevelopmentWebpack"))
"browserDevelopmentWebpack" ->
dependsOn(taskProvider("assemble"))
"browserProductionWebpack" ->
dependsOn(taskProvider("developmentExecutableCompileSync"))
}
}
//// stop warnings when building
//tasks.configureEach {
// when (name) {
// "productionExecutableCompileSync" ->
// dependsOn(taskProvider("compileProductionExecutableKotlinJs"))
// "testTestDevelopmentExecutableCompileSync" ->
// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinJs"))
//// "jsTestTestDevelopmentExecutableCompileSync" ->
//// dependsOn(taskProvider("compileTestDevelopmentExecutableKotlinFrontend"))
//// "jsProductionExecutableCompileSync" ->
//// dependsOn(taskProvider("compileProductionExecutableKotlinFrontend"))
// "browserDevelopmentRun" ->
// dependsOn(taskProvider("browserDevelopmentWebpack"))
// "browserDevelopmentWebpack" ->
// dependsOn(taskProvider("assemble"))
// "browserProductionWebpack" ->
// dependsOn(taskProvider("developmentExecutableCompileSync"))
// }
//}

2
modules/web-map/docker/.env
View file

@ -1,3 +1,3 @@
# Do not edit manually. This file was created with task 'dockerEnv'
COMPOSE_PROJECT_NAME=kafkatorio
KAFKATORIO_VERSION=0.6.0
KAFKATORIO_VERSION=0.6.1

2
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/map.kt
View file

@ -125,7 +125,7 @@ class FactorioMap(
minZoom = -2
maxZoom = 6
maxNativeZoom = -1
maxNativeZoom = 0
minNativeZoom = -1
noWrap = true

6
modules/web-map/src/main/kotlin/dev/adamko/kafkatorio/webmap/websockets.kt
View file

@ -5,6 +5,7 @@ import dev.adamko.kafkatorio.schema.packets.ConsoleChatUpdate
import dev.adamko.kafkatorio.schema.packets.ConsoleCommandUpdate
import dev.adamko.kafkatorio.schema.packets.EntityUpdate
import dev.adamko.kafkatorio.schema.packets.EventServerPacket
import dev.adamko.kafkatorio.schema.packets.KafkatorioPacketData
import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate
import dev.adamko.kafkatorio.schema.packets.PlayerUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
@ -74,7 +75,7 @@ class WebsocketService(
is EventServerPacket.Kafkatorio ->
when (val packetData = packet.packet.data) {
is PlayerUpdate -> reduxStore.dispatch(
is PlayerUpdate -> reduxStore.dispatch(
FactorioUpdate.Player(packet.packet.tick, packetData)
)
is ConfigurationUpdate,
@ -83,7 +84,8 @@ class WebsocketService(
is PrototypesUpdate,
is SurfaceUpdate,
is EntityUpdate,
is MapChunkUpdate -> {
is MapChunkUpdate,
is KafkatorioPacketData.Error -> {
// to be continued...
}
}

10
settings.gradle.kts
View file

@ -10,7 +10,6 @@ include(
":modules:infra-kafka-cluster",
":modules:infra-kafka-pipe",
":modules:infra-kafka-connect",
":modules:kt-rcon",
":modules:web-map",
@ -26,14 +25,5 @@ apply(from = "./buildSrc/repositories.settings.gradle.kts")
@Suppress("UnstableApiUsage") // Central declaration of repositories is an incubating feature
dependencyResolutionManagement {
repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS)
// repositories {
// maven("https://s01.oss.sonatype.org/content/repositories/snapshots/") {
//// content {
//// includeGroup("dev.adamko.kxstsgen")
//// }
// }
// }
}