enable gradle build cache, and improve tasks

buildSrc/build.gradle.kts

@@ -3,7 +3,7 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 plugins {
   idea
   `kotlin-dsl`
-  kotlin("jvm") version "1.6.10"
+  kotlin("jvm") version "1.6.21"
   `project-report`
 }
 
@@ -15,7 +15,7 @@ object Versions {
   const val kotlin = "1.6.21"
 
   const val gradleNodePlugin = "3.2.1"
-  const val gradleDockerComposePlugin = "0.15.1"
+  const val gradleDockerComposePlugin = "0.15.2"
 
   const val kotlinXSerialization = "1.3.2"
   const val kvision = "5.8.2"
@@ -31,6 +31,7 @@ dependencies {
   implementation(enforcedPlatform("org.jetbrains.kotlin:kotlin-bom:${Versions.kotlin}"))
   implementation("org.jetbrains.kotlin:kotlin-serialization")
   implementation("org.jetbrains.kotlin:kotlin-reflect")
+  implementation("org.jetbrains.kotlin:kotlin-compiler-embeddable")
   implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:${Versions.kotlin}")
 
   implementation(platform("org.jetbrains.kotlinx:kotlinx-serialization-bom:${Versions.kotlinXSerialization}"))
@@ -60,7 +61,14 @@ dependencies {
 }
 
 
-tasks.withType<KotlinCompile>().configureEach {
+//tasks.compileKotlin {
+//  kotlinOptions {
+//    // needed by IntelliJ? Even though below does the same?
+//    languageVersion = "1.6"
+//  }
+//}
+
+tasks.withType<KotlinCompile>().all {
 
   kotlinOptions {
     jvmTarget = Versions.jvmTarget

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/gradle/extensions.kt

@@ -1,60 +1,30 @@
 package dev.adamko.kafkatorio.gradle
 
-import dev.adamko.kafkatorio.jsonMapper
 import java.io.ByteArrayOutputStream
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
 import org.gradle.api.Project
-import org.gradle.api.Task
+import org.gradle.api.file.Directory
 import org.gradle.api.file.ProjectLayout
-import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.specs.NotSpec
-import org.gradle.api.provider.MapProperty
 import org.gradle.api.specs.Spec
-import org.gradle.kotlin.dsl.invoke
 import org.gradle.plugins.ide.idea.model.IdeaModule
 import org.gradle.process.ExecSpec
 import org.jetbrains.kotlin.gradle.targets.js.yarn.YarnRootExtension
-
 import org.jetbrains.kotlin.gradle.targets.js.yarn.yarn
-
+import org.jetbrains.kotlin.incremental.md5
-operator fun <T> Spec<T>.not(): Spec<T> = Spec<T> { !this(it) }
 
 
-fun Project.isProcessRunning(process: String, ignoreCase: Boolean = true): Spec<Task> =
+operator fun <T> Spec<T>.not(): NotSpec<T> = NotSpec(this)
-  Spec<Task> {
+//
-    ByteArrayOutputStream().use { outputStream ->
+//
-      project.exec {
+//fun isProcessRunning(process: String, ignoreCase: Boolean = true): Spec<Task> =
-        commandLine("tasklist") // Windows only for now...
+//  Spec<Task> {
-        standardOutput = outputStream
+//    ByteArrayOutputStream().use { outputStream ->
-      }
+//      project.exec {
-      outputStream.toString()
+//      }
-        .contains(process, ignoreCase)
+//      outputStream.toString().contains(process, ignoreCase)
-    }
+//    }
-  }
+//  }
-
+//
-
-fun Project.areJsonPropertiesUpToDate(
-  packageJsonFile: RegularFileProperty,
-  properties: MapProperty<String, String>
-): Spec<Task> = Spec<Task> {
-  val packageJsonContent = packageJsonFile.get().asFile.readText()
-  val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
-
-  properties.get().all { (key, expectedVal) ->
-    val actualVal = packageJson[key]?.jsonPrimitive?.content
-    val isUpToDate = expectedVal == actualVal
-
-    if (isUpToDate) {
-      logger.info("package.json property is up to date, '$key = $actualVal'")
-    } else {
-      logger.lifecycle(
-        "package.json has outdated property '$key' (expected: $expectedVal, actual: $actualVal)"
-      )
-    }
-
-    isUpToDate
-  }
-}
 
 
 /** exclude generated Gradle code, so it doesn't clog up search results */
@@ -91,3 +61,10 @@ fun Project.execCapture(spec: ExecSpec.() -> Unit): String {
 
 /** https://youtrack.jetbrains.com/issue/KT-50848 */
 fun Project.yarn(configure: YarnRootExtension.() -> Unit) = with(yarn, configure)
+
+
+fun Directory.filesChecksum() = asFileTree
+  .files
+  .map { it.readBytes() + it.absolutePath.toByteArray() }
+  .fold(byteArrayOf()) { acc, bytes -> acc + bytes }
+  .md5()

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts

@@ -8,12 +8,6 @@ plugins {
   kotlin("jvm")
 }
 
-kotlin {
-  jvmToolchain {
-    (this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(Versions.jvmTarget))
-  }
-}
-
 dependencies {
 
   implementation(platform("org.jetbrains.kotlin:kotlin-bom"))
@@ -40,9 +34,7 @@ tasks.withType<KotlinCompile>().configureEach {
     jvmTarget = Versions.jvmTarget
     apiVersion = Versions.kotlinLang
     languageVersion = Versions.kotlinLang
-  }
+    freeCompilerArgs += listOf(
-
-  kotlinOptions.freeCompilerArgs += listOf(
 //    "-Xcontext-receivers",
       "-opt-in=kotlin.RequiresOptIn",
       "-opt-in=kotlin.ExperimentalStdlibApi",
@@ -51,11 +43,18 @@ tasks.withType<KotlinCompile>().configureEach {
 //    "-opt-in=kotlinx.serialization.ExperimentalSerializationApi",
     )
   }
+}
 
 tasks.compileTestKotlin {
   kotlinOptions.freeCompilerArgs += "-opt-in=io.kotest.common.ExperimentalKotest"
 }
 
+kotlin {
+  jvmToolchain {
+    (this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(Versions.jvmTarget))
+  }
+}
+
 tasks.withType<Test> {
   useJUnitPlatform()
 }

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/node.gradle.kts

@@ -1,17 +1,8 @@
 package dev.adamko.kafkatorio.lang
 
-import com.github.gradle.node.NodePlugin
 import com.github.gradle.node.npm.task.NpmInstallTask
 import dev.adamko.kafkatorio.Versions
-import dev.adamko.kafkatorio.gradle.areJsonPropertiesUpToDate
-import dev.adamko.kafkatorio.jsonMapper
 import dev.adamko.kafkatorio.task.UpdatePackageJson
-import kotlinx.serialization.encodeToString
-import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.JsonPrimitive
-import kotlinx.serialization.json.jsonObject
-import org.gradle.kotlin.dsl.support.useToRun
-import org.jetbrains.kotlin.util.suffixIfNot
 
 plugins {
   id("com.github.node-gradle.node")
@@ -26,34 +17,34 @@ node {
 }
 
 val updatePackageJson by tasks.registering(UpdatePackageJson::class) {
-  group = NodePlugin.NODE_GROUP
+//  group = NodePlugin.NODE_GROUP
-  description = "Read the package.json file and update the version and name, based on the project."
+//  description = "Read the package.json file and update the version and name, based on the project."
 
-  propertiesToCheck.put("version", "${project.version}")
+  propertiesToCheck.put("version", providers.provider { "${project.version}" })
 
   // check to see if the properties are already up-to-date
-  outputs.upToDateWhen(areJsonPropertiesUpToDate(packageJsonFile, propertiesToCheck))
+//  outputs.upToDateWhen(areJsonPropertiesUpToDate(packageJsonFile, propertiesToCheck))
 //  onlyIf(!areJsonPropertiesUpToDate(packageJsonFile, propertiesToCheck))
 
-  doLast {
+//  doLast {
-    val packageJsonContent = packageJsonFile.get().asFile.readText()
+//    val packageJsonContent = packageJsonFile.get().asFile.readText()
-    val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
+//    val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
-
+//
-    val newJsonProps = propertiesToCheck.get().mapValues { (_, newVal) -> JsonPrimitive(newVal) }
+//    val newJsonProps = propertiesToCheck.get().mapValues { (_, newVal) -> JsonPrimitive(newVal) }
-
+//
-    val packageJsonUpdate = JsonObject(packageJson + newJsonProps)
+//    val packageJsonUpdate = JsonObject(packageJson + newJsonProps)
-    val packageJsonContentUpdated =
+//    val packageJsonContentUpdated =
-      jsonMapper
+//      jsonMapper
-        .encodeToString(packageJsonUpdate)
+//        .encodeToString(packageJsonUpdate)
-        .suffixIfNot("\n")
+//        .suffixIfNot("\n")
-
+//
-    logger.debug(packageJsonContentUpdated)
+//    logger.debug(packageJsonContentUpdated)
-
+//
-    packageJsonFile.get().asFile.writer().useToRun {
+//    packageJsonFile.get().asFile.writer().useToRun {
-      write(packageJsonContentUpdated)
+//      write(packageJsonContentUpdated)
-    }
+//    }
-
+//
-  }
+//  }
 }
 
 tasks.assemble { dependsOn(updatePackageJson) }

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/DockerComposeExec.kt

@@ -0,0 +1,63 @@
+package dev.adamko.kafkatorio.task
+
+import dev.adamko.kafkatorio.gradle.filesChecksum
+import javax.inject.Inject
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.logging.LogLevel
+import org.gradle.api.model.ObjectFactory
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.CacheableTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputDirectory
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.api.tasks.SkipWhenEmpty
+import org.gradle.api.tasks.TaskAction
+import org.gradle.process.ExecOperations
+import org.gradle.work.Incremental
+import org.gradle.work.NormalizeLineEndings
+import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
+
+@CacheableTask
+abstract class DockerComposeExec @Inject constructor(
+  private val executor: ExecOperations,
+  private val objectFactory: ObjectFactory,
+) : DefaultTask() {
+
+  @get:InputDirectory
+  @get:SkipWhenEmpty
+  @get:PathSensitive(PathSensitivity.RELATIVE)
+  @get:NormalizeLineEndings
+  abstract val dockerComposeDir: DirectoryProperty
+
+  @get:Input
+  abstract val command: Property<String>
+
+  @get:OutputFile
+  val stateFile: RegularFileProperty = objectFactory.fileProperty().convention {
+    temporaryDir.resolve("docker-files.md5")
+  }
+
+  init {
+    group = DOCKER_COMPOSE_GROUP
+    logging.captureStandardOutput(LogLevel.LIFECYCLE)
+  }
+
+  @TaskAction
+  fun exec() {
+    executor.exec {
+      workingDir = dockerComposeDir.asFile.get()
+      commandLine = parseSpaceSeparatedArgs(command.get())
+    }
+
+    val inputChecksum = dockerComposeDir.get().filesChecksum()
+    stateFile.get().asFile.writeText("$inputChecksum")
+  }
+
+  companion object {
+    const val DOCKER_COMPOSE_GROUP: String = "docker-compose"
+  }
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/JsonPropertiesUpToDateSpec.kt

@@ -0,0 +1,46 @@
+package dev.adamko.kafkatorio.task
+
+import dev.adamko.kafkatorio.jsonMapper
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+import org.gradle.api.Task
+import org.gradle.api.internal.specs.ExplainingSpec
+
+object JsonPropertiesUpToDateSpec : ExplainingSpec<Task> {
+
+  override fun isSatisfiedBy(element: Task?): Boolean = whyUnsatisfied(element) == null
+
+
+  // Returns a description explaining why the task is outdated,
+  // or null if the task is up to date
+  override fun whyUnsatisfied(element: Task?): String? = with(element) {
+    when (this) {
+      null                  -> "task is null"
+
+      !is UpdatePackageJson ->
+        "task ${this::class.simpleName} is not instance of ${UpdatePackageJson::class.simpleName}"
+
+      else                  -> {
+
+        val packageJsonContent = packageJsonFile.get().asFile.readText()
+        val packageJson = jsonMapper.parseToJsonElement(packageJsonContent).jsonObject
+
+        val outdatedProperties = propertiesToCheck.get().mapNotNull { (key, expectedVal) ->
+          val actualVal = packageJson[key]?.jsonPrimitive?.content
+          when {
+            expectedVal == actualVal -> null
+            else                     -> "'$key' (expected: $expectedVal, actual: $actualVal)"
+          }
+        }
+
+        when {
+          outdatedProperties.isEmpty() -> null
+          outdatedProperties.size == 1 ->
+            "package.json has outdated property: ${outdatedProperties.single()}"
+          else                         ->
+            "package.json has outdated properties: ${outdatedProperties.joinToString(", ")}"
+        }
+      }
+    }
+  }
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/ProcessRunningSpec.kt

@@ -0,0 +1,38 @@
+package dev.adamko.kafkatorio.task
+
+import java.io.ByteArrayOutputStream
+import org.gradle.api.Task
+import org.gradle.api.internal.specs.ExplainingSpec
+import org.gradle.api.tasks.Exec
+
+/** Checks if a process is running. Windows only. */
+class ProcessRunningSpec(
+  private val process: String,
+  private val ignoreCase: Boolean = true,
+) : ExplainingSpec<Task> {
+
+  override fun isSatisfiedBy(element: Task?): Boolean = whyUnsatisfied(element) == null
+
+  override fun whyUnsatisfied(element: Task?): String? = with(element) {
+
+    when (this) {
+      null     -> "task is null"
+      !is Exec -> "task ${this::class.simpleName} is not ${Exec::class.simpleName}"
+      else     -> {
+        return ByteArrayOutputStream().use { outputStream ->
+
+          project.exec {
+            commandLine("tasklist") // Windows only for now...
+            standardOutput = outputStream
+          }
+
+          if (outputStream.toString().contains(process, ignoreCase)) {
+            "found $process in tasklist"
+          } else {
+            null
+          }
+        }
+      }
+    }
+  }
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/TypescriptToLuaTask.kt

@@ -1,28 +1,35 @@
 package dev.adamko.kafkatorio.task
 
 import com.github.gradle.node.npm.task.NpmTask
+import javax.inject.Inject
 import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.FileSystemOperations
 import org.gradle.api.plugins.BasePlugin
+import org.gradle.api.tasks.CacheableTask
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.OutputDirectory
 import org.gradle.api.tasks.PathSensitive
 import org.gradle.api.tasks.PathSensitivity
 import org.gradle.api.tasks.SkipWhenEmpty
+import org.gradle.api.tasks.TaskAction
+import org.gradle.work.Incremental
 import org.gradle.work.NormalizeLineEndings
 import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
 
+@CacheableTask
 abstract class TypescriptToLuaTask : NpmTask() {
 
   @get:InputDirectory
   @get:SkipWhenEmpty
   @get:PathSensitive(PathSensitivity.RELATIVE)
-  @Suppress("UnstableApiUsage")
   @get:NormalizeLineEndings
   abstract val sourceFiles: DirectoryProperty
 
   @get:OutputDirectory
   abstract val outputDirectory: DirectoryProperty
 
+  @get:Inject
+  abstract val fs: FileSystemOperations
 
   init {
     super.setGroup(BasePlugin.BUILD_GROUP)
@@ -33,4 +40,17 @@ abstract class TypescriptToLuaTask : NpmTask() {
     super.args.set(parseSpaceSeparatedArgs("-- --outDir $temporaryDir"))
   }
 
+  @TaskAction
+  fun tstl() {
+    fs.delete { delete(temporaryDir) }
+    temporaryDir.mkdirs()
+
+    super.exec()
+
+    fs.sync {
+      from(temporaryDir)
+      into(outputDirectory)
+    }
+  }
+
 }

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/task/UpdatePackageJson.kt

@@ -1,11 +1,24 @@
 package dev.adamko.kafkatorio.task
 
+import com.github.gradle.node.NodePlugin
+import dev.adamko.kafkatorio.jsonMapper
+import javax.inject.Inject
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+import kotlinx.serialization.json.jsonObject
 import org.gradle.api.DefaultTask
 import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.provider.MapProperty
+import org.gradle.api.provider.ProviderFactory
+import org.gradle.api.tasks.CacheableTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.TaskAction
+import org.gradle.kotlin.dsl.support.useToRun
+import org.jetbrains.kotlin.util.suffixIfNot
 
+@CacheableTask
 abstract class UpdatePackageJson : DefaultTask() {
 
   @get:Input
@@ -14,10 +27,43 @@ abstract class UpdatePackageJson : DefaultTask() {
   @get:OutputFile
   abstract val packageJsonFile: RegularFileProperty
 
+  @get:Inject
+  abstract val providers: ProviderFactory
+//
+//  @get:Inject
+//  abstract val fs: FileSystemOperations
+
   init {
+    group = NodePlugin.NODE_GROUP
+    description = "Read the package.json file and update the version and name, based on the project."
+
+    outputs.upToDateWhen(JsonPropertiesUpToDateSpec)
+
     super.doFirst {
       propertiesToCheck.disallowChanges()
       packageJsonFile.disallowChanges()
     }
   }
+
+  @TaskAction
+  fun exec() {
+    val pj = packageJsonFile.asFile.get()
+
+    logger.info("updating package.json ${pj.canonicalPath}")
+    val packageJson = jsonMapper.parseToJsonElement(pj.readText()).jsonObject
+
+    val newJsonProps = propertiesToCheck.get().mapValues { (_, newVal) -> JsonPrimitive(newVal) }
+
+    val packageJsonUpdate = JsonObject(packageJson + newJsonProps)
+    val packageJsonContentUpdated =
+      jsonMapper
+        .encodeToString(packageJsonUpdate)
+        .suffixIfNot("\n")
+
+    logger.debug(packageJsonContentUpdated)
+
+    packageJsonFile.get().asFile.writer().useToRun {
+      write(packageJsonContentUpdated)
+    }
+  }
 }

gradle.properties

@@ -3,3 +3,9 @@ kotlin.mpp.stability.nowarn=true
 #kotlin.mpp.enableGranularSourceSetsMetadata=true
 #kotlin.native.enableDependencyPropagation=false
 org.gradle.dependency.verification.console=verbose
+
+org.gradle.caching=true
+org.gradle.unsafe.configuration-cache=true
+org.gradle.unsafe.configuration-cache-problems=warn
+
+org.gradle.kotlin.dsl.precompiled.accessors.strict=true

modules/events-mod/build.gradle.kts

@@ -39,6 +39,7 @@ projectTokens += mapOf(
 
 val tsSrcDir: Directory = layout.projectDirectory.dir("src/main/typescript")
 
+
 node {
   nodeProjectDir.set(tsSrcDir)
 }
@@ -49,27 +50,19 @@ val typescriptEventsSchema: Configuration by configurations.creating {
   typescriptAttributes(objects)
 }
 
+
 dependencies {
   typescriptEventsSchema(projects.modules.eventsSchema)
 }
 
+
 val typescriptToLua by tasks.registering(TypescriptToLuaTask::class) {
   dependsOn(tasks.npmInstall, installEventsTsSchema, tasks.updatePackageJson)
 
   sourceFiles.set(tsSrcDir)
   outputDirectory.set(layout.buildDirectory.dir("typescriptToLua"))
+}
 
-  doFirst("cleanTemporaryDir") {
-    delete(temporaryDir)
-    mkdir(temporaryDir)
-  }
-  doLast("syncTstlOutput") {
-    sync {
-      from(temporaryDir)
-      into(outputDirectory)
-    }
-  }
-}
 
 val installEventsTsSchema by tasks.registering(Sync::class) {
   description = "Fetch the latest shared data-model"
@@ -100,10 +93,13 @@ val installEventsTsSchema by tasks.registering(Sync::class) {
   into(outputDir)
 }
 
+
 tasks.distZip {
-  archiveFileName.set(distributionZipName)
+  archiveFileName.set(provider { distributionZipName })
+
 }
 
+
 distributions {
   main {
 
@@ -165,11 +161,13 @@ val factorioModProvider by configurations.registering {
   outgoing.artifact(tasks.distZip.flatMap { it.archiveFile })
 }
 
+
 tasks.updatePackageJson {
-  propertiesToCheck.put("name", "${rootProject.name}-${project.name}")
+  propertiesToCheck.put("name", providers.provider { "${rootProject.name}-${project.name}" })
   packageJsonFile.set(layout.projectDirectory.file("src/main/typescript/package.json"))
 }
 
+
 tasks.assemble { dependsOn(installEventsTsSchema, tasks.updatePackageJson) }
 
 

modules/events-mod/src/main/typescript/package.json

@@ -9,8 +9,7 @@
   "engines": {
     "node": "~14.19.1"
   },
-  "devDependencies": {
+  "devDependencies": {},
-  },
   "dependencies": {
     "lua-types": "^2.11.0",
     "typescript-to-lua": "1.4.3",
@@ -19,10 +18,7 @@
   },
   "// @types/node": "^12.0.8",
   "license": "UNLICENSED",
-  "peerDependencies": {
+  "peerDependencies": {},
-  },
+  "optionalDependencies": {},
-  "optionalDependencies": {
+  "bundledDependencies": []
-  },
-  "bundledDependencies": [
-  ]
 }

modules/events-schema/build.gradle.kts

@@ -1,6 +1,5 @@
 import dev.adamko.kafkatorio.gradle.asProvider
 import dev.adamko.kafkatorio.gradle.typescriptAttributes
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 
 plugins {
@@ -111,11 +110,16 @@ kotlin {
 val generateTypescript by tasks.registering(JavaExec::class) {
   group = "kt2ts"
 
-  val jvmJar by tasks.getting(Jar::class)
+  val jvmJar = tasks.named<Jar>(kotlin.jvm().artifactsTaskName)
-  val main by kotlin.jvm().compilations.getting
+  dependsOn(jvmJar)
+
+  val mainCompilation = kotlin.jvm().compilations.named("main").map { it.runtimeDependencyFiles }
+//  dependsOn(mainCompilation)
+  inputs.files(mainCompilation)
+
   classpath(
     jvmJar,
-    main.runtimeDependencyFiles
+    mainCompilation
   )
 
 //  mainClass.set("dev.adamko.kafkatorio.events.schema.Kt2tsKt")

modules/events-server/src/main/kotlin/dev/adamko/kafkatorio/processor/topology/groupMapChunks.kt

@@ -13,6 +13,7 @@ import dev.adamko.kafkatorio.schema.common.toMapChunkPosition
 import dev.adamko.kafkatorio.schema.packets.MapChunkUpdate
 import dev.adamko.kafkatorio.schema.packets.PrototypesUpdate
 import dev.adamko.kotka.extensions.groupedAs
+import dev.adamko.kotka.extensions.materializedAs
 import dev.adamko.kotka.extensions.materializedWith
 import dev.adamko.kotka.extensions.producedAs
 import dev.adamko.kotka.extensions.streams.filter
@@ -158,74 +159,15 @@ private fun groupTilesIntoChunksWithColours(
           kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
         )
       ) { chunkTiles, other -> chunkTiles + other }
-//      .toStream("server-map-data.tiles.repartition.to-stream")
-//      .repartition(
-//        repartitionedAs(
-//          "server-map-data.tiles.repartition.config",
-//          kxsBinary.serde<ServerMapChunkId>(),
-//          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
-//          3,
-//        )
-//      ).toTable(
-//        "server-map-data.tiles.repartition.to-table",
-//        materializedWith(
-//          kxsBinary.serde<ServerMapChunkId>(),
-//          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>?>(),
-//        )
-//      )
-//      .filter("server-map-data.tiles.repartition.filter-not-null") { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
-//        v != null
-//      }
-//      .mapValues(
-//        "server-map-data.tiles.repartition.map-not-null",
-//        materializedWith(
-//          kxsBinary.serde<ServerMapChunkId>(),
-//          kxsBinary.serde<ServerMapChunkTiles<TileProtoHashCode>>(),
-//        )
-//      ) { _: ServerMapChunkId, v: ServerMapChunkTiles<TileProtoHashCode>? ->
-//        v as ServerMapChunkTiles<TileProtoHashCode>
-//      }
-
-
-  // repartition to try and fix KTable-KTable joining skipping records
-  val tileProtoColourDictRepartitioned =
-    tileProtoColourDict
-//      .toStream("tileProtoColourDict.repartition.to-stream")
-//      .repartition(
-//        repartitionedAs(
-//          "tileProtoColourDict.repartition.config",
-//          kxsBinary.serde<FactorioServerId>(),
-//          kxsBinary.serde<TileColourDict?>(),
-//          3,
-//        )
-//      ).toTable(
-//        "tileProtoColourDict.repartition.to-table",
-//        materializedWith(
-//          kxsBinary.serde<FactorioServerId>(),
-//          kxsBinary.serde<TileColourDict?>(),
-//        )
-//      )
-//      .filter("tileProtoColourDict.repartition.filter-not-null") { _: FactorioServerId, v: TileColourDict? ->
-//        v != null
-//      }
-//      .mapValues(
-//        "tileProtoColourDict.repartition.map-not-null",
-//        materializedWith(
-//          kxsBinary.serde<FactorioServerId>(),
-//          kxsBinary.serde<TileColourDict>(),
-//        )
-//      ) { _: FactorioServerId, v: TileColourDict? ->
-//        v as TileColourDict
-//      }
 
 
   val colourisedChunkTable: KTable<ServerMapChunkId, ServerMapChunkTiles<ColourHex>> =
     chunkedTilesTable
       .join(
-        other = tileProtoColourDictRepartitioned,
+        other = tileProtoColourDict,
         tableJoined = tableJoined("server-map-data.join-tiles-with-prototypes"),
-        materialized = materializedWith(
+        materialized = materializedAs(
-//          "server-map-data.join-tiles-with-prototypes.store",
+          "server-map-data.join-tiles-with-prototypes.store",
           kxsBinary.serde<ServerMapChunkId>(),
           kxsBinary.serde<ServerMapChunkTiles<ColourHex>>(),
         ),

modules/infra-factorio-client/build.gradle.kts

@@ -1,7 +1,7 @@
 import dev.adamko.kafkatorio.gradle.asConsumer
 import dev.adamko.kafkatorio.gradle.factorioModAttributes
-import dev.adamko.kafkatorio.gradle.isProcessRunning
 import dev.adamko.kafkatorio.gradle.not
+import dev.adamko.kafkatorio.task.ProcessRunningSpec
 import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
 
 plugins {
@@ -38,7 +38,7 @@ val deployModToClient by tasks.registering(Copy::class) {
 //</editor-fold>
 
 //<editor-fold desc="Factorio client lifecycle tasks">
-fun isFactorioRunning(): Spec<Task> = isProcessRunning("factorio.exe")
+fun isFactorioRunning(): Spec<Task> = ProcessRunningSpec("factorio.exe")
 
 val clientLaunch by tasks.registering(Exec::class) {
   description = "Run Factorio Steam game client"

modules/infra-kafka-pipe/build.gradle.kts

@@ -1,4 +1,3 @@
-import org.jetbrains.kotlin.incremental.md5
 import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
 
 plugins {
@@ -9,35 +8,32 @@ description = "Send events from a Factorio server to a Kafka topic "
 
 
 val dockerSrcDir: Directory by extra
-fun Directory.filesChecksum() = asFileTree
-  .files
-  .map { it.readBytes() + it.absolutePath.toByteArray() }
-  .fold(byteArrayOf()) { acc, bytes -> acc + bytes }
-  .md5()
 
 tasks.dockerUp {
   dependsOn(":modules:infra-kafka-cluster:dockerUp")
 }
 
-val dockerBuildKafkaPipe by tasks.registering(Exec::class) {
-  group = "docker-compose"
 
-  logging.captureStandardOutput(LogLevel.LIFECYCLE)
 
+@Suppress("UnstableApiUsage") // normalizeLineEndings is incubating
+val dockerBuildKafkaPipe by tasks.registering(dev.adamko.kafkatorio.task.DockerComposeExec::class) {
   dependsOn(tasks.dockerEnv)
 
-  inputs.dir(dockerSrcDir)
+  dockerComposeDir.set(dockerSrcDir)
-  inputs.property("dockerSrcDirChecksum") { dockerSrcDir.filesChecksum() }
+  command.set("docker-compose build kafka-pipe")
 
-  outputs.upToDateWhen {
+
-    it.inputs.properties["dockerSrcDirChecksum"] as? Long? == dockerSrcDir.filesChecksum()
+//  inputs.property("dockerSrcDirChecksum") { dockerSrcDir.filesChecksum() }
-  }
+//
+//  outputs.upToDateWhen {
+//    it.inputs.properties["dockerSrcDirChecksum"] as? Long? == dockerSrcDir.filesChecksum()
+//  }
 
 //  val imageIdFile = file("$temporaryDir/docker-image-id.txt")
 //  outputs.file(imageIdFile)
 
-  workingDir = dockerSrcDir.asFile
+//  workingDir = dockerSrcDir.asFile
-  commandLine = parseSpaceSeparatedArgs(""" docker-compose build kafka-pipe """)
+//  commandLine = parseSpaceSeparatedArgs(""" docker-compose build kafka-pipe """)
 
 //  doLast {
 //    sync {
@@ -53,4 +49,5 @@ val dockerBuildKafkaPipe by tasks.registering(Exec::class) {
 //    }
 //}
 }
+
 tasks.assemble { dependsOn(dockerBuildKafkaPipe) }

settings.gradle.kts

@@ -18,7 +18,6 @@ include(
   ":modules:factorio-mod-portal-client",
 )
 
-enableFeaturePreview("VERSION_CATALOGS")
 enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS")
 
 apply(from = "./buildSrc/repositories.settings.gradle.kts")