lazzexus/Kafkatorio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

enable gradle build cache, and improve tasks

Browse source
This commit is contained in:
Adam 2022-04-23 17:59:37 +02:00
parent f339333f76
commit b81e62e2fd
17 changed files with 330 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

14
buildSrc/build.gradle.kts
View file

@ -3,7 +3,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
idea
`kotlin-dsl`
kotlin("jvm") version "1.6.10"
kotlin("jvm") version "1.6.21"
`project-report`
}
@ -15,7 +15,7 @@ object Versions {
const val kotlin = "1.6.21"
const val gradleNodePlugin = "3.2.1"
const val gradleDockerComposePlugin = "0.15.1"
const val gradleDockerComposePlugin = "0.15.2"
const val kotlinXSerialization = "1.3.2"
const val kvision = "5.8.2"
@ -31,6 +31,7 @@ dependencies {
implementation(enforcedPlatform("org.jetbrains.kotlin:kotlin-bom:${Versions.kotlin}"))
implementation("org.jetbrains.kotlin:kotlin-serialization")
implementation("org.jetbrains.kotlin:kotlin-reflect")
implementation("org.jetbrains.kotlin:kotlin-compiler-embeddable")
implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:${Versions.kotlin}")
implementation(platform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}"))
@ -60,7 +61,14 @@ dependencies {
}
tasks.withType<KotlinCompile>().configureEach {
//tasks.compileKotlin {
// kotlinOptions {
// // needed by IntelliJ? Even though below does the same?
// languageVersion = "1.6"
// }
//}
tasks.withType<KotlinCompile>().all {
kotlinOptions {
jvmTarget = Versions.jvmTarget

67
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/gradle/extensions.kt
View file

@ -1,60 +1,30 @@
package dev.adamko.kafkatorio.gradle
import dev.adamko.kafkatorio.jsonMapper
import java.io.ByteArrayOutputStream
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.Directory
import org.gradle.api.file.ProjectLayout
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.MapProperty
import org.gradle.api.specs.NotSpec
import org.gradle.api.specs.Spec
import org.gradle.kotlin.dsl.invoke
import org.gradle.plugins.ide.idea.model.IdeaModule
import org.gradle.process.ExecSpec
import org.jetbrains.kotlin.gradle.targets.js.yarn.YarnRootExtension
import org.jetbrains.kotlin.gradle.targets.js.yarn.yarn
operator fun <T> Spec<T>.not(): Spec<T> = Spec<T> { !this(it) }
import org.jetbrains.kotlin.incremental.md5
fun Project.isProcessRunning(process: String, ignoreCase: Boolean = true): Spec<Task> =
Spec<Task> {
ByteArrayOutputStream().use { outputStream ->
project.exec {
commandLine("tasklist") // Windows only for now...
standardOutput = outputStream
}
outputStream.toString()
.contains(process, ignoreCase)
}
}
fun Project.areJsonPropertiesUpToDate(
packageJsonFile: RegularFileProperty,
properties: MapProperty<String, String>
): Spec<Task> = Spec<Task> {
val packageJsonContent = packageJsonFile.get().asFile.readText()
val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
properties.get().all { (key, expectedVal) ->
val actualVal = packageJson[key]?.jsonPrimitive?.content
val isUpToDate = expectedVal == actualVal
if (isUpToDate) {
logger.info("package.json property is up to date, '$key = $actualVal'")
} else {
logger.lifecycle(
"package.json has outdated property '$key' (expected: $expectedVal, actual: $actualVal)"
)
}
isUpToDate
}
}
operator fun <T> Spec<T>.not(): NotSpec<T> = NotSpec(this)
//
//
//fun isProcessRunning(process: String, ignoreCase: Boolean = true): Spec<Task> =
// Spec<Task> {
// ByteArrayOutputStream().use { outputStream ->
// project.exec {
// }
// outputStream.toString().contains(process, ignoreCase)
// }
// }
//
/** exclude generated Gradle code, so it doesn't clog up search results */
@ -91,3 +61,10 @@ fun Project.execCapture(spec: ExecSpec.() -> Unit): String {
/** https://youtrack.jetbrains.com/issue/KT-50848 */
fun Project.yarn(configure: YarnRootExtension.() -> Unit) = with(yarn, configure)
fun Directory.filesChecksum() = asFileTree
.files
.map { it.readBytes() + it.absolutePath.toByteArray() }
.fold(byteArrayOf()) { acc, bytes -> acc + bytes }
.md5()

17
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts
View file

@ -8,12 +8,6 @@ plugins {
kotlin("jvm")
}
kotlin {
jvmToolchain {
(this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(Versions.jvmTarget))
}
}
dependencies {
implementation(platform("org.jetbrains.kotlin:kotlin-bom"))
@ -40,9 +34,7 @@ tasks.withType<KotlinCompile>().configureEach {
jvmTarget = Versions.jvmTarget
apiVersion = Versions.kotlinLang
languageVersion = Versions.kotlinLang
}
kotlinOptions.freeCompilerArgs += listOf(
freeCompilerArgs += listOf(
// "-Xcontext-receivers",
"-opt-in=kotlin.RequiresOptIn",
"-opt-in=kotlin.ExperimentalStdlibApi",
@ -50,12 +42,19 @@ tasks.withType<KotlinCompile>().configureEach {
// "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
// "-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
)
}
}
tasks.compileTestKotlin {
kotlinOptions.freeCompilerArgs += "-opt-in=io.kotest.common.ExperimentalKotest"
}
kotlin {
jvmToolchain {
(this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(Versions.jvmTarget))
}
}
tasks.withType<Test> {
useJUnitPlatform()
}

55
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/node.gradle.kts
View file

@ -1,17 +1,8 @@
package dev.adamko.kafkatorio.lang
import com.github.gradle.node.NodePlugin
import com.github.gradle.node.npm.task.NpmInstallTask
import dev.adamko.kafkatorio.Versions
import dev.adamko.kafkatorio.gradle.areJsonPropertiesUpToDate
import dev.adamko.kafkatorio.jsonMapper
import dev.adamko.kafkatorio.task.UpdatePackageJson
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.jsonObject
import org.gradle.kotlin.dsl.support.useToRun
import org.jetbrains.kotlin.util.suffixIfNot
plugins {
id("com.github.node-gradle.node")
@ -26,34 +17,34 @@ node {
}
val updatePackageJson by tasks.registering(UpdatePackageJson::class) {
group = NodePlugin.NODE_GROUP
description = "Read the package.json file and update the version and name, based on the project."
// group = NodePlugin.NODE_GROUP
// description = "Read the package.json file and update the version and name, based on the project."
propertiesToCheck.put("version", "${project.version}")
propertiesToCheck.put("version", providers.provider { "${project.version}" })
// check to see if the properties are already up-to-date
outputs.upToDateWhen(areJsonPropertiesUpToDate(packageJsonFile, propertiesToCheck))
// outputs.upToDateWhen(areJsonPropertiesUpToDate(packageJsonFile, propertiesToCheck))
// onlyIf(!areJsonPropertiesUpToDate(packageJsonFile, propertiesToCheck))
doLast {
val packageJsonContent = packageJsonFile.get().asFile.readText()
val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
val newJsonProps = propertiesToCheck.get().mapValues { (_, newVal) -> JsonPrimitive(newVal) }
val packageJsonUpdate = JsonObject(packageJson + newJsonProps)
val packageJsonContentUpdated =
jsonMapper
.encodeToString(packageJsonUpdate)
.suffixIfNot("\n")
logger.debug(packageJsonContentUpdated)
packageJsonFile.get().asFile.writer().useToRun {
write(packageJsonContentUpdated)
}
}
// doLast {
// val packageJsonContent = packageJsonFile.get().asFile.readText()
// val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
//
// val newJsonProps = propertiesToCheck.get().mapValues { (_, newVal) -> JsonPrimitive(newVal) }
//
// val packageJsonUpdate = JsonObject(packageJson + newJsonProps)
// val packageJsonContentUpdated =
// jsonMapper
// .encodeToString(packageJsonUpdate)
// .suffixIfNot("\n")
//
// logger.debug(packageJsonContentUpdated)
//
// packageJsonFile.get().asFile.writer().useToRun {
// write(packageJsonContentUpdated)
// }
//
// }
}
tasks.assemble { dependsOn(updatePackageJson) }

63
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/DockerComposeExec.kt Normal file
View file

@ -0,0 +1,63 @@
package dev.adamko.kafkatorio.task
import dev.adamko.kafkatorio.gradle.filesChecksum
import javax.inject.Inject
import org.gradle.api.DefaultTask
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.logging.LogLevel
import org.gradle.api.model.ObjectFactory
import org.gradle.api.provider.Property
import org.gradle.api.tasks.CacheableTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.PathSensitive
import org.gradle.api.tasks.PathSensitivity
import org.gradle.api.tasks.SkipWhenEmpty
import org.gradle.api.tasks.TaskAction
import org.gradle.process.ExecOperations
import org.gradle.work.Incremental
import org.gradle.work.NormalizeLineEndings
import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
@CacheableTask
abstract class DockerComposeExec @Inject constructor(
private val executor: ExecOperations,
private val objectFactory: ObjectFactory,
) : DefaultTask() {
@get:InputDirectory
@get:SkipWhenEmpty
@get:PathSensitive(PathSensitivity.RELATIVE)
@get:NormalizeLineEndings
abstract val dockerComposeDir: DirectoryProperty
@get:Input
abstract val command: Property<String>
@get:OutputFile
val stateFile: RegularFileProperty = objectFactory.fileProperty().convention {
temporaryDir.resolve("docker-files.md5")
}
init {
group = DOCKER_COMPOSE_GROUP
logging.captureStandardOutput(LogLevel.LIFECYCLE)
}
@TaskAction
fun exec() {
executor.exec {
workingDir = dockerComposeDir.asFile.get()
commandLine = parseSpaceSeparatedArgs(command.get())
}
val inputChecksum = dockerComposeDir.get().filesChecksum()
stateFile.get().asFile.writeText("$inputChecksum")
}
companion object {
const val DOCKER_COMPOSE_GROUP: String = "docker-compose"
}
}

46
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/JsonPropertiesUpToDateSpec.kt Normal file
View file

@ -0,0 +1,46 @@
package dev.adamko.kafkatorio.task
import dev.adamko.kafkatorio.jsonMapper
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import org.gradle.api.Task
import org.gradle.api.internal.specs.ExplainingSpec
object JsonPropertiesUpToDateSpec : ExplainingSpec<Task> {
override fun isSatisfiedBy(element: Task?): Boolean = whyUnsatisfied(element) == null
// Returns a description explaining why the task is outdated,
// or null if the task is up to date
override fun whyUnsatisfied(element: Task?): String? = with(element) {
when (this) {
null -> "task is null"
!is UpdatePackageJson ->
"task ${this::class.simpleName} is not instance of ${UpdatePackageJson::class.simpleName}"
else -> {
val packageJsonContent = packageJsonFile.get().asFile.readText()
val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
val outdatedProperties = propertiesToCheck.get().mapNotNull { (key, expectedVal) ->
val actualVal = packageJson[key]?.jsonPrimitive?.content
when {
expectedVal == actualVal -> null
else -> "'$key' (expected: $expectedVal, actual: $actualVal)"
}
}
when {
outdatedProperties.isEmpty() -> null
outdatedProperties.size == 1 ->
"package.json has outdated property: ${outdatedProperties.single()}"
else ->
"package.json has outdated properties: ${outdatedProperties.joinToString(", ")}"
}
}
}
}
}

38
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/ProcessRunningSpec.kt Normal file
View file

@ -0,0 +1,38 @@
package dev.adamko.kafkatorio.task
import java.io.ByteArrayOutputStream
import org.gradle.api.Task
import org.gradle.api.internal.specs.ExplainingSpec
import org.gradle.api.tasks.Exec
/** Checks if a process is running. Windows only. */
class ProcessRunningSpec(
private val process: String,
private val ignoreCase: Boolean = true,
) : ExplainingSpec<Task> {
override fun isSatisfiedBy(element: Task?): Boolean = whyUnsatisfied(element) == null
override fun whyUnsatisfied(element: Task?): String? = with(element) {
when (this) {
null -> "task is null"
!is Exec -> "task ${this::class.simpleName} is not ${Exec::class.simpleName}"
else -> {
return ByteArrayOutputStream().use { outputStream ->
project.exec {
commandLine("tasklist") // Windows only for now...
standardOutput = outputStream
}
if (outputStream.toString().contains(process, ignoreCase)) {
"found $process in tasklist"
} else {
null
}
}
}
}
}
}

22
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/TypescriptToLuaTask.kt
View file

@ -1,28 +1,35 @@
package dev.adamko.kafkatorio.task
import com.github.gradle.node.npm.task.NpmTask
import javax.inject.Inject
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.file.FileSystemOperations
import org.gradle.api.plugins.BasePlugin
import org.gradle.api.tasks.CacheableTask
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.PathSensitive
import org.gradle.api.tasks.PathSensitivity
import org.gradle.api.tasks.SkipWhenEmpty
import org.gradle.api.tasks.TaskAction
import org.gradle.work.Incremental
import org.gradle.work.NormalizeLineEndings
import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
@CacheableTask
abstract class TypescriptToLuaTask : NpmTask() {
@get:InputDirectory
@get:SkipWhenEmpty
@get:PathSensitive(PathSensitivity.RELATIVE)
@Suppress("UnstableApiUsage")
@get:NormalizeLineEndings
abstract val sourceFiles: DirectoryProperty
@get:OutputDirectory
abstract val outputDirectory: DirectoryProperty
@get:Inject
abstract val fs: FileSystemOperations
init {
super.setGroup(BasePlugin.BUILD_GROUP)
@ -33,4 +40,17 @@ abstract class TypescriptToLuaTask : NpmTask() {
super.args.set(parseSpaceSeparatedArgs("-- --outDir $temporaryDir"))
}
@TaskAction
fun tstl() {
fs.delete { delete(temporaryDir) }
temporaryDir.mkdirs()
super.exec()
fs.sync {
from(temporaryDir)
into(outputDirectory)
}
}
}

46
buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/UpdatePackageJson.kt
View file

@ -1,11 +1,24 @@
package dev.adamko.kafkatorio.task
import com.github.gradle.node.NodePlugin
import dev.adamko.kafkatorio.jsonMapper
import javax.inject.Inject
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.jsonObject
import org.gradle.api.DefaultTask
import org.gradle.api.file.RegularFileProperty
import org.gradle.api.provider.MapProperty
import org.gradle.api.provider.ProviderFactory
import org.gradle.api.tasks.CacheableTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import org.gradle.kotlin.dsl.support.useToRun
import org.jetbrains.kotlin.util.suffixIfNot
@CacheableTask
abstract class UpdatePackageJson : DefaultTask() {
@get:Input
@ -14,10 +27,43 @@ abstract class UpdatePackageJson : DefaultTask() {
@get:OutputFile
abstract val packageJsonFile: RegularFileProperty
@get:Inject
abstract val providers: ProviderFactory
//
// @get:Inject
// abstract val fs: FileSystemOperations
init {
group = NodePlugin.NODE_GROUP
description = "Read the package.json file and update the version and name, based on the project."
outputs.upToDateWhen(JsonPropertiesUpToDateSpec)
super.doFirst {
propertiesToCheck.disallowChanges()
packageJsonFile.disallowChanges()
}
}
@TaskAction
fun exec() {
val pj = packageJsonFile.asFile.get()
logger.info("updating package.json ${pj.canonicalPath}")
val packageJson = jsonMapper.parseToJsonElement(pj.readText()).jsonObject
val newJsonProps = propertiesToCheck.get().mapValues { (_, newVal) -> JsonPrimitive(newVal) }
val packageJsonUpdate = JsonObject(packageJson + newJsonProps)
val packageJsonContentUpdated =
jsonMapper
.encodeToString(packageJsonUpdate)
.suffixIfNot("\n")
logger.debug(packageJsonContentUpdated)
packageJsonFile.get().asFile.writer().useToRun {
write(packageJsonContentUpdated)
}
}
}

6
gradle.properties
View file

@ -3,3 +3,9 @@ kotlin.mpp.stability.nowarn=true
#kotlin.mpp.enableGranularSourceSetsMetadata=true
#kotlin.native.enableDependencyPropagation=false
org.gradle.dependency.verification.console=verbose
org.gradle.caching=true
org.gradle.unsafe.configuration-cache=true
org.gradle.unsafe.configuration-cache-problems=warn
org.gradle.kotlin.dsl.precompiled.accessors.strict=true

24
modules/events-mod/build.gradle.kts
View file

@ -39,6 +39,7 @@ projectTokens += mapOf(
val tsSrcDir: Directory = layout.projectDirectory.dir("src/main/typescript")
node {
nodeProjectDir.set(tsSrcDir)
}
@ -49,28 +50,20 @@ val typescriptEventsSchema: Configuration by configurations.creating {
typescriptAttributes(objects)
}
dependencies {
typescriptEventsSchema(projects.modules.eventsSchema)
}
val typescriptToLua by tasks.registering(TypescriptToLuaTask::class) {
dependsOn(tasks.npmInstall, installEventsTsSchema, tasks.updatePackageJson)
sourceFiles.set(tsSrcDir)
outputDirectory.set(layout.buildDirectory.dir("typescriptToLua"))
doFirst("cleanTemporaryDir") {
delete(temporaryDir)
mkdir(temporaryDir)
}
doLast("syncTstlOutput") {
sync {
from(temporaryDir)
into(outputDirectory)
}
}
}
val installEventsTsSchema by tasks.registering(Sync::class) {
description = "Fetch the latest shared data-model"
group = project.name
@ -100,10 +93,13 @@ val installEventsTsSchema by tasks.registering(Sync::class) {
into(outputDir)
}
tasks.distZip {
archiveFileName.set(distributionZipName)
archiveFileName.set(provider { distributionZipName })
}
distributions {
main {
@ -165,11 +161,13 @@ val factorioModProvider by configurations.registering {
outgoing.artifact(tasks.distZip.flatMap { it.archiveFile })
}
tasks.updatePackageJson {
propertiesToCheck.put("name", "${rootProject.name}-${project.name}")
propertiesToCheck.put("name", providers.provider { "${rootProject.name}-${project.name}" })
packageJsonFile.set(layout.projectDirectory.file("src/main/typescript/package.json"))
}
tasks.assemble { dependsOn(installEventsTsSchema, tasks.updatePackageJson) }

12
modules/events-mod/src/main/typescript/package.json
View file

@ -9,8 +9,7 @@
"engines": {
"node": "~14.19.1"
},
"devDependencies": {
},
"devDependencies": {},
"dependencies": {
"lua-types": "^2.11.0",
"typescript-to-lua": "1.4.3",
@ -19,10 +18,7 @@
},
"// @types/node": "^12.0.8",
"license": "UNLICENSED",
"peerDependencies": {
},
"optionalDependencies": {
},
"bundledDependencies": [
]
"peerDependencies": {},
"optionalDependencies": {},
"bundledDependencies": []
}

12
modules/events-schema/build.gradle.kts
View file

@ -1,6 +1,5 @@
import dev.adamko.kafkatorio.gradle.asProvider
import dev.adamko.kafkatorio.gradle.typescriptAttributes
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
@ -111,11 +110,16 @@ kotlin {
val generateTypescript by tasks.registering(JavaExec::class) {
group = "kt2ts"
val jvmJar by tasks.getting(Jar::class)
val main by kotlin.jvm().compilations.getting
val jvmJar = tasks.named<Jar>(kotlin.jvm().artifactsTaskName)
dependsOn(jvmJar)
val mainCompilation = kotlin.jvm().compilations.named("main").map { it.runtimeDependencyFiles }
// dependsOn(mainCompilation)
inputs.files(mainCompilation)
classpath(
jvmJar,
main.runtimeDependencyFiles
mainCompilation
)
// mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2tsKt")

66
modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt
View file

@ -13,6 +13,7 @@ import dev.adamko.kafkatorio.schema.common.toMapChunkPosition
import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate
import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
import dev.adamko.kotka.extensions.groupedAs
import dev.adamko.kotka.extensions.materializedAs
import dev.adamko.kotka.extensions.materializedWith
import dev.adamko.kotka.extensions.producedAs
import dev.adamko.kotka.extensions.streams.filter
@ -158,74 +159,15 @@ private fun groupTilesIntoChunksWithColours(
kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
)
) { chunkTiles, other -> chunkTiles + other }
// .toStream("server-map-data.tiles.repartition.to-stream")
// .repartition(
// repartitionedAs(
// "server-map-data.tiles.repartition.config",
// kxsBinary.serde<ServerMapChunkId>(),
// kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
// 3,
// )
// ).toTable(
// "server-map-data.tiles.repartition.to-table",
// materializedWith(
// kxsBinary.serde<ServerMapChunkId>(),
// kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
// )
// )
// .filter("server-map-data.tiles.repartition.filter-not-null") { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
// v != null
// }
// .mapValues(
// "server-map-data.tiles.repartition.map-not-null",
// materializedWith(
// kxsBinary.serde<ServerMapChunkId>(),
// kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
// )
// ) { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
// v as ServerMapChunkTiles<TileProtoHashCode>
// }
// repartition to try and fix KTable-KTable joining skipping records
val tileProtoColourDictRepartitioned =
tileProtoColourDict
// .toStream("tileProtoColourDict.repartition.to-stream")
// .repartition(
// repartitionedAs(
// "tileProtoColourDict.repartition.config",
// kxsBinary.serde<FactorioServerId>(),
// kxsBinary.serde<TileColourDict?>(),
// 3,
// )
// ).toTable(
// "tileProtoColourDict.repartition.to-table",
// materializedWith(
// kxsBinary.serde<FactorioServerId>(),
// kxsBinary.serde<TileColourDict?>(),
// )
// )
// .filter("tileProtoColourDict.repartition.filter-not-null") { _: FactorioServerId, v: TileColourDict? ->
// v != null
// }
// .mapValues(
// "tileProtoColourDict.repartition.map-not-null",
// materializedWith(
// kxsBinary.serde<FactorioServerId>(),
// kxsBinary.serde<TileColourDict>(),
// )
// ) { _: FactorioServerId, v: TileColourDict? ->
// v as TileColourDict
// }
val colourisedChunkTable: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
chunkedTilesTable
.join(
other = tileProtoColourDictRepartitioned,
other = tileProtoColourDict,
tableJoined = tableJoined("server-map-data.join-tiles-with-prototypes"),
materialized = materializedWith(
// "server-map-data.join-tiles-with-prototypes.store",
materialized = materializedAs(
"server-map-data.join-tiles-with-prototypes.store",
kxsBinary.serde<ServerMapChunkId>(),
kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
),

4
modules/infra-factorio-client/build.gradle.kts
View file

@ -1,7 +1,7 @@
import dev.adamko.kafkatorio.gradle.asConsumer
import dev.adamko.kafkatorio.gradle.factorioModAttributes
import dev.adamko.kafkatorio.gradle.isProcessRunning
import dev.adamko.kafkatorio.gradle.not
import dev.adamko.kafkatorio.task.ProcessRunningSpec
import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
plugins {
@ -38,7 +38,7 @@ val deployModToClient by tasks.registering(Copy::class) {
//</editor-fold>
//<editor-fold desc="Factorio client lifecycle tasks">
fun isFactorioRunning(): Spec<Task> = isProcessRunning("factorio.exe")
fun isFactorioRunning(): Spec<Task> = ProcessRunningSpec("factorio.exe")
val clientLaunch by tasks.registering(Exec::class) {
description = "Run Factorio Steam game client"

29
modules/infra-kafka-pipe/build.gradle.kts
View file

@ -1,4 +1,3 @@
import org.jetbrains.kotlin.incremental.md5
import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
plugins {
@ -9,35 +8,32 @@ description = "Send events from a Factorio server to a Kafka topic "
val dockerSrcDir: Directory by extra
fun Directory.filesChecksum() = asFileTree
.files
.map { it.readBytes() + it.absolutePath.toByteArray() }
.fold(byteArrayOf()) { acc, bytes -> acc + bytes }
.md5()
tasks.dockerUp {
dependsOn(":modules:infra-kafka-cluster:dockerUp")
}
val dockerBuildKafkaPipe by tasks.registering(Exec::class) {
group = "docker-compose"
logging.captureStandardOutput(LogLevel.LIFECYCLE)
@Suppress("UnstableApiUsage") // normalizeLineEndings is incubating
val dockerBuildKafkaPipe by tasks.registering(dev.adamko.kafkatorio.task.DockerComposeExec::class) {
dependsOn(tasks.dockerEnv)
inputs.dir(dockerSrcDir)
inputs.property("dockerSrcDirChecksum") { dockerSrcDir.filesChecksum() }
dockerComposeDir.set(dockerSrcDir)
command.set("docker-compose build kafka-pipe")
outputs.upToDateWhen {
it.inputs.properties["dockerSrcDirChecksum"] as? Long? == dockerSrcDir.filesChecksum()
}
// inputs.property("dockerSrcDirChecksum") { dockerSrcDir.filesChecksum() }
//
// outputs.upToDateWhen {
// it.inputs.properties["dockerSrcDirChecksum"] as? Long? == dockerSrcDir.filesChecksum()
// }
// val imageIdFile = file("$temporaryDir/docker-image-id.txt")
// outputs.file(imageIdFile)
workingDir = dockerSrcDir.asFile
commandLine = parseSpaceSeparatedArgs(""" docker-compose build kafka-pipe """)
// workingDir = dockerSrcDir.asFile
// commandLine = parseSpaceSeparatedArgs(""" docker-compose build kafka-pipe """)
// doLast {
// sync {
@ -53,4 +49,5 @@ val dockerBuildKafkaPipe by tasks.registering(Exec::class) {
// }
//}
}
tasks.assemble { dependsOn(dockerBuildKafkaPipe) }

1
settings.gradle.kts
View file

@ -18,7 +18,6 @@ include(
":modules:factorio-mod-portal-client",
)
enableFeaturePreview("VERSION_CATALOGS")
enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
apply(from = "./buildSrc/repositories.settings.gradle.kts")