[geedeecee] improve handling of build-context files, and build-avoidance, and remove old code

gradle-plugins/builder-plugins/src/main/kotlin/kafkatorio/extensions/gradle.kt

@@ -80,12 +80,12 @@ fun Directory.filesChecksum(): Long = asFileTree
 
 
 /** Drop the first [count] directories from the path */
-fun FileCopyDetails.dropDirectories(count: Int): RelativePath =
-  RelativePath(true, *relativePath.segments.drop(count).toTypedArray())
+fun RelativePath.dropDirectories(count: Int): RelativePath =
+  RelativePath(true, *segments.drop(count).toTypedArray())
 
 
 /** Drop the first directory from the path */
-fun FileCopyDetails.dropDirectory(): RelativePath =
+fun RelativePath.dropDirectory(): RelativePath =
   dropDirectories(1)
 
 

gradle-plugins/docker-compose/build.gradle.kts

@@ -1,31 +1,13 @@
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
-
 plugins {
   id("kafkatorio.conventions.kotlin-dsl")
-//  kotlin("jvm") version embeddedKotlinVersion
-//  `kotlin-dsl`
   `java-gradle-plugin`
 }
 
-val gradleJvmTarget = "11"
-
 dependencies {
   implementation(platform(libs.kotlin.bom))
   implementation("org.jetbrains.kotlin:kotlin-compiler-embeddable")
 }
 
-//kotlin {
-//  jvmToolchain {
-//    (this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(gradleJvmTarget))
-//  }
-//}
-//
-//tasks.withType<KotlinCompile>().configureEach {
-//  kotlinOptions {
-//    jvmTarget = gradleJvmTarget
-//  }
-//}
-
 gradlePlugin {
   plugins {
     create("geedeecee") {

gradle-plugins/docker-compose/src/main/kotlin/dev/adamko/geedeecee/DockerComposeExec.kt

@@ -1,96 +0,0 @@
-package dev.adamko.geedeecee
-
-import javax.inject.Inject
-import org.gradle.api.DefaultTask
-import org.gradle.api.file.Directory
-import org.gradle.api.file.DirectoryProperty
-import org.gradle.api.file.RegularFileProperty
-import org.gradle.api.logging.LogLevel
-import org.gradle.api.model.ObjectFactory
-import org.gradle.api.provider.Property
-import org.gradle.api.provider.ProviderFactory
-import org.gradle.api.tasks.CacheableTask
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputDirectory
-import org.gradle.api.tasks.OutputFile
-import org.gradle.api.tasks.PathSensitive
-import org.gradle.api.tasks.PathSensitivity
-import org.gradle.api.tasks.SkipWhenEmpty
-import org.gradle.api.tasks.TaskAction
-import org.gradle.kotlin.dsl.property
-import org.gradle.process.ExecOperations
-import org.gradle.work.NormalizeLineEndings
-import org.jetbrains.kotlin.incremental.md5
-import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
-
-
-// look into using 'build context' for up-to-date check
-// https://stackoverflow.com/questions/38946683/how-to-test-dockerignore-file
-// https://github.com/pwaller/docker-show-context
-// https://snippets.khromov.se/see-which-files-are-included-in-your-docker-build-context/
-
-
-@CacheableTask
-@Suppress("UnstableApiUsage")
-abstract class DockerComposeExec @Inject constructor(
-  private val executor: ExecOperations,
-  private val objects: ObjectFactory,
-  private val providers: ProviderFactory,
-) : DefaultTask() {
-
-  @get:InputDirectory
-  @get:SkipWhenEmpty
-  @get:PathSensitive(PathSensitivity.RELATIVE)
-  @get:NormalizeLineEndings
-  abstract val dockerComposeDir: DirectoryProperty
-
-  @get:Input
-  abstract val command: Property<String>
-
-  @get:Input
-  val dockerIsActive: Property<Boolean> = objects.property<Boolean>().convention(
-    providers.exec {
-      commandLine = parseSpaceSeparatedArgs("docker info")
-      isIgnoreExitValue = true
-    }.result.map {
-      it.exitValue == 0
-    }
-  )
-
-  @get:OutputFile
-  val stateFile: RegularFileProperty = objects.fileProperty().convention {
-    temporaryDir.resolve("docker-files.md5")
-  }
-
-  init {
-    group = DOCKER_COMPOSE_GROUP
-    logging.captureStandardOutput(LogLevel.LIFECYCLE)
-
-    super.onlyIf { task ->
-      require(task is DockerComposeExec)
-      task.dockerIsActive.getOrElse(false)
-    }
-  }
-
-  @TaskAction
-  fun exec() {
-    executor.exec {
-      workingDir = dockerComposeDir.asFile.get()
-      commandLine = parseSpaceSeparatedArgs(command.get())
-    }
-
-    val inputChecksum = dockerComposeDir.get().filesChecksum()
-    stateFile.get().asFile.writeText("$inputChecksum")
-  }
-
-  companion object {
-    const val DOCKER_COMPOSE_GROUP: String = "docker-compose"
-
-
-    private fun Directory.filesChecksum(): Long = asFileTree
-      .files
-      .map { it.readBytes() + it.absolutePath.toByteArray() }
-      .fold(byteArrayOf()) { acc, bytes -> acc + bytes }
-      .md5()
-  }
-}

gradle-plugins/docker-compose/src/main/kotlin/dev/adamko/geedeecee/GDCCommandTask.kt

@@ -1,66 +0,0 @@
-package dev.adamko.geedeecee
-
-import javax.inject.Inject
-import org.gradle.api.DefaultTask
-import org.gradle.api.Task
-import org.gradle.api.file.DirectoryProperty
-import org.gradle.api.logging.LogLevel
-import org.gradle.api.provider.ListProperty
-import org.gradle.api.provider.Property
-import org.gradle.api.tasks.Input
-import org.gradle.api.tasks.InputDirectory
-import org.gradle.api.tasks.TaskAction
-import org.gradle.process.ExecOperations
-import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
-
-abstract class GDCCommandTask @Inject constructor(
-  private val executor: ExecOperations
-) : DefaultTask() {
-
-  @get:InputDirectory
-  abstract val workingDir: DirectoryProperty
-
-  @get:Input
-  abstract val separatedArgs: ListProperty<String>
-
-  @get:Input
-  abstract val dockerActive: Property<Boolean>
-
-  init {
-    super.setGroup(GDCPlugin.GCD_TASK_GROUP)
-    description = "Run a docker-compose command."
-
-    enabledIf {
-      it.dockerActive.orNull == true
-    }
-
-    logging.captureStandardOutput(LogLevel.LIFECYCLE)
-  }
-
-  @TaskAction
-  fun dockerCompose() {
-    val separatedArgs: List<String> = separatedArgs.get()
-
-    executor.exec {
-      environment("BUILDKIT_PROGRESS", "plain")
-      workingDir(this@GDCCommandTask.workingDir)
-      commandLine(separatedArgs)
-    }
-  }
-
-  @Suppress("FunctionName")
-  fun `docker-compose`(args: String) {
-    separatedArgs.addAll(
-      parseSpaceSeparatedArgs("docker-compose $args")
-    )
-  }
-
-  companion object {
-    private inline fun <reified T: Task> T.enabledIf(crossinline spec: (T) -> Boolean) {
-      onlyIf {
-        require(it is T)
-        spec(it)
-      }
-    }
-  }
-}

gradle-plugins/docker-compose/src/main/kotlin/dev/adamko/geedeecee/GDCPlugin.kt

@@ -1,12 +1,19 @@
 package dev.adamko.geedeecee
 
+import dev.adamko.geedeecee.tasks.DockerContextFilesPreparation
+import dev.adamko.geedeecee.tasks.DockerEnvUpdateTask
+import dev.adamko.geedeecee.tasks.GDCCommandTask
 import javax.inject.Inject
-import org.gradle.api.*
-import org.gradle.api.file.ProjectLayout
-import org.gradle.api.logging.*
-import org.gradle.api.provider.*
-import org.gradle.api.tasks.*
-import org.gradle.kotlin.dsl.*
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.provider.Provider
+import org.gradle.api.provider.ProviderFactory
+import org.gradle.kotlin.dsl.apply
+import org.gradle.kotlin.dsl.create
+import org.gradle.kotlin.dsl.getValue
+import org.gradle.kotlin.dsl.provideDelegate
+import org.gradle.kotlin.dsl.registering
+import org.gradle.kotlin.dsl.withType
 import org.gradle.language.base.plugins.LifecycleBasePlugin
 import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
 
@@ -14,7 +21,6 @@ import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
 @Suppress("UnstableApiUsage")
 abstract class GDCPlugin @Inject constructor(
   private val providers: ProviderFactory,
-  private val layout: ProjectLayout,
 ) : Plugin<Project> {
 
 
@@ -34,6 +40,14 @@ abstract class GDCPlugin @Inject constructor(
 //      )
     }
 
+    val dockerContextPrepareFiles by target.tasks.registering(DockerContextFilesPreparation::class) {
+      into(gdcSettings.dockerBuildContextDir)
+      includeEmptyDirs = false
+//      dockerContextDir.set(gdcSettings.dockerBuildContextDir)
+//      copySpec.convention(target.copySpec())
+//      copySpec = Action {  }
+    }
+
     val dockerComposeUp by target.tasks.registering(GDCCommandTask::class) {
       `docker-compose`("up -d")
     }
@@ -45,6 +59,7 @@ abstract class GDCPlugin @Inject constructor(
     }
     val dockerComposeBuild by target.tasks.registering(GDCCommandTask::class) {
       `docker-compose`("build")
+      cacheable.set(true)
     }
     val dockerComposePush by target.tasks.registering(GDCCommandTask::class) {
       dependsOn(dockerComposeBuild)
@@ -57,14 +72,21 @@ abstract class GDCPlugin @Inject constructor(
     dockerComposeUp.configure { dependsOn(dockerComposeBuild) }
     dockerComposeRemove.configure { dependsOn(dockerComposeDown) }
 
-    target.tasks.withType<DockerComposeExec>().configureEach {
-      dependsOn(dockerComposeEnvUpdate)
-    }
+//    target.tasks.withType<DockerComposeExec>().configureEach {
+//      dependsOn(dockerComposeEnvUpdate)
+//      dockerIsActive.set(isDockerActive())
+//      stateFile.set(temporaryDir.resolve("docker-state.md5"))
+//    }
 
     target.tasks.withType<GDCCommandTask>().configureEach {
-      workingDir.convention(gdcSettings.srcDir)
       dependsOn(dockerComposeEnvUpdate)
+      dependsOn(dockerContextPrepareFiles)
+
+      workingDir.convention(gdcSettings.srcDir)
       dockerActive.set(gdcSettings.dockerActive)
+      stateFile.set(
+        gdcSettings.stateDir.file(workingDir.map { "dc_state${name.hashCode()}${it.asFile.hashCode()}.md5" })
+      )
     }
 
     val assembleTasks = target.tasks.matching { LifecycleBasePlugin.ASSEMBLE_TASK_NAME == it.name }
@@ -76,7 +98,6 @@ abstract class GDCPlugin @Inject constructor(
     dockerComposeUp.configure {
       dependsOn(assembleTasks)
     }
-
   }
 
 
@@ -86,6 +107,8 @@ abstract class GDCPlugin @Inject constructor(
       composeProjectVersion.convention(providers.provider { project.version.toString() })
       containerRegistryHost.convention(providers.gradleProperty("dockerContainerRegistryHost"))
       srcDir.convention(layout.projectDirectory.dir("docker"))
+      dockerBuildContextDir.convention(srcDir.dir("build"))
+      stateDir.convention(layout.buildDirectory.dir("geedeecee/state/"))
 
       dotEnv.put("COMPOSE_PROJECT_NAME", composeProjectName)
       dotEnv.put("APP_NAME", providers.provider { project.name })

gradle-plugins/docker-compose/src/main/kotlin/dev/adamko/geedeecee/GDCSettings.kt

@@ -26,6 +26,11 @@ interface GDCSettings {
   @get:Input
   val dockerActive: Property<Boolean>
 
+  @get:Input
+  val dockerBuildContextDir: DirectoryProperty
+
+  val stateDir: DirectoryProperty
+
 //  @Suppress("PropertyName", "FunctionName")
 //  abstract class DotEnv(
 //    private val dotEnv: MapProperty<String, String>,

gradle-plugins/docker-compose/src/main/kotlin/dev/adamko/geedeecee/tasks/DockerEnvUpdateTask.kt

@@ -1,5 +1,6 @@
-package dev.adamko.geedeecee
+package dev.adamko.geedeecee.tasks
 
+import dev.adamko.geedeecee.GDCPlugin
 import java.io.File
 import java.nio.charset.Charset
 import java.util.Properties

gradle-plugins/docker-compose/src/main/kotlin/dev/adamko/geedeecee/tasks/GDCCommandTask.kt

@@ -0,0 +1,90 @@
+package dev.adamko.geedeecee.tasks
+
+import dev.adamko.geedeecee.GDCPlugin
+import dev.adamko.geedeecee.internal.enabledIf
+import dev.adamko.geedeecee.internal.filesChecksum
+import javax.inject.Inject
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.logging.LogLevel
+import org.gradle.api.provider.ListProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputDirectory
+import org.gradle.api.tasks.Optional
+import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.api.tasks.SkipWhenEmpty
+import org.gradle.api.tasks.TaskAction
+import org.gradle.process.ExecOperations
+import org.gradle.work.NormalizeLineEndings
+import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs
+
+
+// look into using 'build context' for up-to-date check
+// https://stackoverflow.com/questions/38946683/how-to-test-dockerignore-file
+// https://github.com/pwaller/docker-show-context
+// https://snippets.khromov.se/see-which-files-are-included-in-your-docker-build-context/
+// or parsing Dockerfile
+// https://github.com/keilerkonzept/dockerfile-json
+
+
+@Suppress("UnstableApiUsage") // NormalizeLineEndings
+abstract class GDCCommandTask @Inject constructor(
+  private val executor: ExecOperations,
+) : DefaultTask() {
+
+  @get:InputDirectory
+  @get:SkipWhenEmpty
+  @get:PathSensitive(PathSensitivity.RELATIVE)
+  @get:NormalizeLineEndings
+  abstract val workingDir: DirectoryProperty
+
+  @get:Input
+  abstract val separatedArgs: ListProperty<String>
+
+  @get:Input
+  abstract val dockerActive: Property<Boolean>
+
+  @get:Input
+  @get:Optional
+  abstract val cacheable: Property<Boolean>
+
+  @get:OutputFile
+  abstract val stateFile: RegularFileProperty
+
+  init {
+    group = GDCPlugin.GCD_TASK_GROUP
+    description = "Run a docker-compose command."
+
+    enabledIf { it.dockerActive.getOrElse(false) }
+
+    logging.captureStandardOutput(LogLevel.LIFECYCLE)
+  }
+
+  @TaskAction
+  fun dockerCompose() {
+    val separatedArgs: List<String> = separatedArgs.get()
+
+    executor.exec {
+      environment("BUILDKIT_PROGRESS", "plain")
+      workingDir(this@GDCCommandTask.workingDir)
+      commandLine(separatedArgs)
+    }
+
+    if (cacheable.getOrElse(false)) {
+      val inputChecksum = workingDir.get().filesChecksum()
+      stateFile.get().asFile.writeText("$inputChecksum")
+    }
+  }
+
+  /** Set the `docker-compose` argument */
+  @Suppress("FunctionName")
+  fun `docker-compose`(args: String) {
+    separatedArgs.set(
+      parseSpaceSeparatedArgs("docker-compose $args")
+    )
+  }
+}