- version bumps for coroutines, gradle, kotlin

- refactor kafka-connector project, prepping for the connector-admin

build.gradle.kts

@@ -27,7 +27,7 @@ idea {
 }
 
 tasks.wrapper {
-  gradleVersion = "7.3.1"
+  gradleVersion = "7.3.3"
   distributionType = Wrapper.DistributionType.ALL
 }
 tasks.assemble { dependsOn(tasks.wrapper) }
@@ -38,6 +38,7 @@ val startInfra by tasks.registering {
   dependsOn(
     ":modules:infra-kafka-pipe:processRun",
     ":modules:infra-kafka-cluster:processRun",
+    ":modules:infra-kafka-connect:processRun",
   )
 }
 

buildSrc/build.gradle.kts

@@ -3,13 +3,13 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 plugins {
   idea
   `kotlin-dsl`
-  kotlin("jvm") version "1.6.0"
+  kotlin("jvm") version "1.6.10"
   `project-report`
 }
 
 dependencies {
 
-  val kotlinVersion = "1.6.0"
+  val kotlinVersion = "1.6.10"
   implementation(enforcedPlatform("org.jetbrains.kotlin:kotlin-bom:$kotlinVersion"))
   implementation("org.jetbrains.kotlin:kotlin-serialization")
   implementation("org.jetbrains.kotlin:kotlin-reflect")
@@ -19,7 +19,7 @@ dependencies {
   implementation(enforcedPlatform("org.jetbrains.kotlinx:kotlinx-serialization-bom:$kotlinXSerializationVersion"))
   implementation("org.jetbrains.kotlinx:kotlinx-serialization-core")
   implementation("org.jetbrains.kotlinx:kotlinx-serialization-json")
-
+//  implementation("com.charleskorn.kaml:kaml:0.37.0")
 
   implementation("com.github.node-gradle:gradle-node-plugin:3.1.1")
 
@@ -40,7 +40,7 @@ tasks.withType<KotlinCompile>().configureEach {
     "-Xopt-in=kotlin.RequiresOptIn",
     "-Xopt-in=kotlin.ExperimentalStdlibApi",
     "-Xopt-in=kotlin.time.ExperimentalTime",
-    "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
+//    "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
     "-Xopt-in=kotlinx.serialization.ExperimentalSerializationApi",
   )
 }

buildSrc/repositories.settings.gradle.kts

@@ -1,4 +1,3 @@
-
 @Suppress("UnstableApiUsage")
 dependencyResolutionManagement {
 
@@ -13,7 +12,7 @@ dependencyResolutionManagement {
   repositories {
     mavenCentral()
     gradlePluginPortal()
-    maven("https://jitpack.io")
+    jitpack()
     maven("https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev/")
     maven("https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven")
 
@@ -35,11 +34,16 @@ dependencyResolutionManagement {
   pluginManagement {
 
     repositories {
-      mavenCentral()
       gradlePluginPortal()
+      mavenCentral()
+      jitpack()
       maven("https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev/")
     }
 
   }
 
 }
+
+fun RepositoryHandler.jitpack() {
+  maven("https://jitpack.io")
+}

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/common.kt

@@ -2,7 +2,6 @@ package dev.adamko.kafkatorio
 
 import kotlinx.serialization.json.Json
 
-
 val jsonMapper = Json {
   prettyPrint = true
   prettyPrintIndent = "  "

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/infra/docker-compose.gradle.kts

@@ -8,23 +8,26 @@ plugins {
 
 val dockerComposeTaskGroup: String by extra("docker-compose")
 
-val composeProjectName: String by extra(rootProject.name)
-val srcDir: Directory by extra(layout.projectDirectory.dir("src"))
+val dockerComposeProjectName: String by extra(rootProject.name)
+val dockerSrcDir: Directory by extra(layout.projectDirectory.dir("src"))
 
 
-val dockerUp by tasks.creating(Exec::class) {
+val dockerUp by tasks.registering(Exec::class) {
   group = dockerComposeTaskGroup
 
   dependsOn(tasks.assemble)
+  logging.captureStandardOutput(LogLevel.LIFECYCLE)
 
-  workingDir = srcDir.asFile
+  workingDir = dockerSrcDir.asFile
   commandLine = parseSpaceSeparatedArgs(""" docker-compose up -d """)
 }
 
-val dockerDown by tasks.creating(Exec::class) {
+val dockerDown by tasks.registering(Exec::class) {
   group = dockerComposeTaskGroup
 
-  workingDir = srcDir.asFile
+  logging.captureStandardOutput(LogLevel.LIFECYCLE)
+
+  workingDir = dockerSrcDir.asFile
   commandLine = parseSpaceSeparatedArgs(""" docker-compose down """)
 }
 
@@ -36,9 +39,11 @@ afterEvaluate {
 val dockerEnv by tasks.registering(WriteProperties::class) {
   group = dockerComposeTaskGroup
 
-  setOutputFile(srcDir.file(".env"))
+  logging.captureStandardOutput(LogLevel.LIFECYCLE)
+
+  setOutputFile(dockerSrcDir.file(".env"))
   properties(
-    "COMPOSE_PROJECT_NAME" to composeProjectName,
+    "COMPOSE_PROJECT_NAME" to dockerComposeProjectName,
   )
 }
 

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-js.gradle.kts

@@ -23,7 +23,7 @@ kotlin {
 }
 
 dependencies {
-  val kotlinWrappersVersion = "0.0.1-pre.270-kotlin-1.6.0"
+  val kotlinWrappersVersion = "0.0.1-pre.276-kotlin-1.6.0"
   implementation(
     project.dependencies.enforcedPlatform(
       "org.jetbrains.kotlin-wrappers:kotlin-wrappers-bom:${kotlinWrappersVersion}"

buildSrc/src/main/kotlin/dev/adamko/kafkatorio/lang/kotlin-jvm.gradle.kts

@@ -1,8 +1,6 @@
 package dev.adamko.kafkatorio.lang
 
-import org.gradle.kotlin.dsl.dependencies
 import org.gradle.kotlin.dsl.kotlin
-import org.gradle.kotlin.dsl.withType
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
@@ -29,11 +27,12 @@ dependencies {
 
   val junitVersion = "5.8.2"
   testImplementation(enforcedPlatform("org.junit:junit-bom:$junitVersion"))
-  testImplementation("org.junit.jupiter:junit-jupiter-api")
-  testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
-  testRuntimeOnly("org.junit.platform:junit-platform-launcher")
+  testImplementation("org.junit.jupiter:junit-jupiter")
+  testRuntimeOnly("org.junit.platform:junit-platform-launcher") {
+    because("Only needed to run tests in a version of IntelliJ IDEA that bundles older versions")
+  }
 
-  val kotestVersion = "5.0.1"
+  val kotestVersion = "5.0.3"
   testImplementation(enforcedPlatform("io.kotest:kotest-bom:$kotestVersion"))
   testImplementation("io.kotest:kotest-runner-junit5:$kotestVersion")
   testImplementation("io.kotest:kotest-assertions-core:$kotestVersion")
@@ -58,7 +57,7 @@ tasks.withType<KotlinCompile>().configureEach {
     "-Xopt-in=kotlin.RequiresOptIn",
     "-Xopt-in=kotlin.ExperimentalStdlibApi",
     "-Xopt-in=kotlin.time.ExperimentalTime",
-    "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
+//    "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi",
     "-Xopt-in=kotlinx.serialization.ExperimentalSerializationApi",
   )
 }
@@ -66,3 +65,7 @@ tasks.withType<KotlinCompile>().configureEach {
 tasks.compileTestKotlin {
   kotlinOptions.freeCompilerArgs += "-Xopt-in=io.kotest.common.ExperimentalKotest"
 }
+
+tasks.withType<Test> {
+  useJUnitPlatform()
+}

gradle/libs.versions.toml

@@ -6,11 +6,11 @@
 
 ## compilation ##
 jvm        = "11"
-kotlin     = "1.6.0"
+kotlin     = "1.6.10"
 
 ## libs ##
 camel-kafkaConnectors = "0.11.0"
-http4k                = "4.17.2.0"
+http4k                = "4.17.4.0"
 kafka                 = "3.0.0"
 kotlinx-serialization = "1.3.1"
 kvision               = "5.6.1"
@@ -51,7 +51,7 @@ kotlinx-serialization-bom  = { group = "org.jetbrains.kotlinx", name = "kotlinx-
 kotlinx-serialization-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-core" }
 kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json" }
 kotlinx-serialization-yaml = { group = "com.charleskorn.kaml",  name = "kaml",                       version = "0.37.0"    }
-kotlinx-coroutines         = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core",    version = "1.6.0-RC" }
+kotlinx-coroutines         = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core",    version = "1.6.0" }
 kotlinx-html               = { group = "org.jetbrains.kotlinx", name = "kotlinx-html",               version = "0.7.3"     }
 
 ## http4k ##

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

modules/events-schema/build.gradle.kts

@@ -22,7 +22,7 @@ kotlin {
         optIn("kotlin.RequiresOptIn")
         optIn("kotlin.ExperimentalStdlibApi")
         optIn("kotlin.time.ExperimentalTime")
-        optIn("kotlinx.coroutines.ExperimentalCoroutinesApi")
+//        optIn("kotlinx.coroutines.ExperimentalCoroutinesApi")
         optIn("kotlinx.serialization.ExperimentalSerializationApi")
         optIn("kotlin.js.ExperimentalJsExport")
       }

modules/infra-factorio-server/build.gradle.kts

@@ -6,8 +6,8 @@ plugins {
   id("dev.adamko.kafkatorio.infra.docker-compose")
 }
 
-val srcDir: Directory by extra
-val factorioServerDataDir: Directory = srcDir.dir("factorio-server")
+val dockerSrcDir: Directory by extra
+val factorioServerDataDir: Directory = dockerSrcDir.dir("factorio-server")
 
 val factorioMod: Configuration by configurations.creating {
   asConsumer()

modules/infra-kafka-connect/build.gradle.kts

@@ -1,15 +1,15 @@
+@file:Suppress("UnstableApiUsage") // VERSION_PLATFORM + platform(...) is unstable
+
 import dev.adamko.kafkatorio.gradle.asConsumer
-import dev.adamko.kafkatorio.jsonMapper
-import kotlinx.serialization.encodeToString
-import kotlinx.serialization.json.contentOrNull
-import kotlinx.serialization.json.jsonArray
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
 
 plugins {
   id("dev.adamko.kafkatorio.infra.docker-compose")
+  id("dev.adamko.kafkatorio.lang.kotlin-jvm")
 }
 
+val dockerSrcDir: Directory by extra(layout.projectDirectory.dir("src/main/docker"))
+val connectorJarDir: Directory = dockerSrcDir.dir("camel-kafka-connectors")
+
 val camelConnectorDependencies: Configuration by configurations.creating { asConsumer() }
 
 fun DependencyHandlerScope.kafkaConnector(
@@ -23,11 +23,19 @@ fun DependencyHandlerScope.kafkaConnector(
 dependencies {
   kafkaConnector("camel-websocket-kafka-connector")
   kafkaConnector("camel-ahc-ws-kafka-connector")
+
+  implementation("com.github.adamko-dev:json5-kotlin:2.0.3")
+
+  implementation(platform(libs.http4k.bom))
+  implementation("org.http4k:http4k-core")
+  implementation("org.http4k:http4k-client-okhttp")
+
+  implementation(platform(libs.kotlinx.serialization.bom))
+  implementation("org.jetbrains.kotlinx:kotlinx-serialization-core")
+  implementation("org.jetbrains.kotlinx:kotlinx-serialization-json")
 }
 
 
-val connectorJarDir: Directory = layout.projectDirectory.dir("src/camel-kafka-connectors")
-
 val downloadCamelConnectors by tasks.registering(Sync::class) {
   group = project.name
   description = "Retrieve Camel Kafka Connectors from Maven, in preparation for the Docker build"
@@ -44,7 +52,6 @@ val downloadCamelConnectors by tasks.registering(Sync::class) {
   into(connectorJarDir)
 }
 
-
 tasks.dockerUp {
   dependsOn(":modules:infra-kafka-cluster:dockerUp")
   dependsOn(downloadCamelConnectors)
@@ -56,72 +63,3 @@ tasks.dockerEnv {
     "KAFKA_CONNECT_VERSION" to libs.versions.confluent.kafkaConnect.get(),
   )
 }
-
-
-val availableCamelVersions by tasks.registering {
-  // dicking around with some sort of 'all camel connectors' docker image.
-  // it's unfinished.
-
-  val camelGroup = "org.apache.camel.kafkaconnector"
-  val camelParent = "connectors"
-
-  val requestUrl = """
-    https://search.maven.org/solrsearch/select?q=g:"$camelGroup"+AND+a:"$camelParent"&core=gav&rows=20&wt=json
-  """.trimIndent()
-    .replace("\"", "%22")
-
-  doLast {
-
-    val connectorVersions = java.net.URL(requestUrl).readText()
-    logger.lifecycle("conn ==== $connectorVersions")
-    val versionsJson = jsonMapper.parseToJsonElement(connectorVersions).jsonObject
-
-    logger.lifecycle("json ==== \n ${jsonMapper.encodeToString(versionsJson)}")
-
-    val deps = versionsJson["response"]
-      ?.jsonObject
-      ?.get("docs")
-      ?.jsonArray
-      ?.map { it.jsonObject }
-      ?.mapNotNull { j ->
-        val group = j["g"]?.jsonPrimitive?.contentOrNull
-        val artifact = j["a"]?.jsonPrimitive?.contentOrNull
-        val version = j["v"]?.jsonPrimitive?.contentOrNull
-        val packaging = j["p"]?.jsonPrimitive?.contentOrNull?.toLowerCase()
-
-        if (listOf(group, artifact, version, packaging).all { it != null }) {
-          val d = project.dependencies.create(
-            group = group!!,
-            name = artifact!!,
-            version = version!!,
-            ext = packaging,
-          )
-          logger.lifecycle("dep = $d / $packaging")
-          d
-        } else {
-          null
-        }
-      } ?: emptyList()
-
-    val c = configurations.detachedConfiguration()
-    c.dependencies.addAll(deps)
-    c.resolve()
-      .joinToString("\n\n---\n\n") { it.readLines().take(5).joinToString("\n") }
-
-//    logger.lifecycle("ids: ${versions?.joinToString("\n", "\n\n", "\n\n")}")
-//
-//    versions?.filterNotNull()?.forEach { id ->
-//
-//      configurations.detachedConfiguration(
-//        project.dependencies.create(id)
-//      )
-//        .resolve()
-//        .joinToString("\n\n---\n\n") { it.readLines().take(5).joinToString("\n") }
-//
-//    }
-
-
-//  jsonMapper.parseToJsonElement()
-
-  }
-}

modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt

@@ -0,0 +1,51 @@
+package dev.adamko.kafkatorio.kafkaconnect
+
+import at.syntaxerror.json5.Json5Module
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonObject
+
+import kotlinx.serialization.json.Json
+
+val jsonMapper = Json {
+  prettyPrint = true
+  prettyPrintIndent = "  "
+}
+
+val j5 = Json5Module()
+
+//val updateConnectors by tasks.registering {
+//  group = project.name
+//
+//  val configFile = layout.projectDirectory.file("configs/websocket-sink-lua-topics.json5")
+//  inputs.file(configFile)
+//
+//  doLast {
+//    val fileText = resources.text.fromFile(configFile)
+//
+//    val jsonObject: JsonObject = j5.decodeObject(fileText.asString())
+//
+//    logger.lifecycle("out: $jsonObject")
+//    val configJson = jsonMapper.encodeToString(jsonObject)
+//    logger.lifecycle("configJson: $configJson")
+//
+//    val request = Request(
+//      PUT,
+//      Uri.of("http://localhost:8083").path("/connectors/kafkatorio-ws-connector/config")
+//    ).header("Content-Type", "application/json")
+//      .body(configJson)
+//
+//
+//    val client = OkHttp()
+//    val response = client(request)
+//
+//    println(response.toMessage())
+//    println(
+//      jsonMapper.encodeToString(
+//        jsonMapper.decodeFromString<JsonElement>(response.body.toString())
+//      )
+//    )
+//  }
+//
+//}

modules/infra-kafka-connect/src/main/resources/configs/websocket-sink-lua-topics.json5

@@ -0,0 +1,64 @@
+//Exchange data with external Websocket servers using Async Http Client
+{
+  name: "CKC-async-ws-lua-objects",
+  "connector.class": "org.apache.camel.kafkaconnector.ahcws.CamelAhcwsSinkConnector",
+  "tasks.max": 1,
+  //use the kafka converters that better suit your needs, these are just defaults:
+  "key.converter": "org.apache.kafka.connect.storage.StringConverter",
+  "value.converter": "org.apache.kafka.connect.storage.StringConverter",
+  //https://docs.confluent.io/platform/current/installation/configuration/connect/sink-connect-configs.html#sinkconnectorconfigs_topics
+  //List of topics to consume, separated by commas
+  // topics=
+  "topics.regex": "^Lua.*$",
+
+  // mandatory properties (for a complete properties list see the connector documentation):
+
+  // The URI to use such as http://hostname:port/path
+  "camel.sink.path.httpUri": "http://localhost:/ws",
+  // If the option is true, then the Exchange.HTTP_URI header is ignored, and use the endpoint's URI for request. You may also set the throwExceptionOnFailure to be false to let the AhcProducer send all the fault response back.
+  // camel.sink.endpoint.bridgeEndpoint=
+  // The initial in-memory buffer size used when transferring data between Camel and AHC Client.
+  // camel.sink.endpoint.bufferSize=
+  // To use a custom HeaderFilterStrategy to filter header to and from Camel message.
+  // camel.sink.endpoint.headerFilterStrategy=
+  // Option to disable throwing the AhcOperationFailedException in case of failed responses from the remote server. This allows you to get all responses regardless of the HTTP status code.
+  // camel.sink.endpoint.throwExceptionOnFailure=
+  // If enabled and an Exchange failed processing on the consumer side, and if the caused Exception was send back serialized in the response as a application\/x-java-serialized-object content type (for example using Jetty or Servlet Camel components). On the producer side the exception will be deserialized and thrown as is, instead of the AhcOperationFailedException. The caused exception is required to be serialized. This is by default turned off. If you enable this then be aware that Java will deserialize the incoming data from the request to Java and that can be a potential security risk.
+  // camel.sink.endpoint.transferException=
+  // Define if the Connection Close header has to be added to HTTP Request. This parameter is false by default
+  // camel.sink.endpoint.connectionClose=
+  // Configure a cookie handler to maintain a HTTP session
+  // camel.sink.endpoint.cookieHandler=
+  // Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail during starting and cause the route to fail being started. By deferring this startup to be lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. Beware that when the first message is processed then creating and starting the producer may take a little time and prolong the total processing time of the processing.
+  // camel.sink.endpoint.lazyStartProducer=
+  // To enable streaming to send data as multiple text fragments.
+  // camel.sink.endpoint.useStreaming=
+  // To use a custom AhcBinding which allows to control how to bind between AHC and Camel.
+  // camel.sink.endpoint.binding=
+  // To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance.
+  // camel.sink.endpoint.clientConfig=
+  // To configure the AsyncHttpClientConfig using the key\/values from the Map.
+  // camel.sink.endpoint.clientConfigOptions=
+  // To configure the AsyncHttpClientConfig Realm using the key\/values from the Map.
+  // camel.sink.endpoint.clientConfigRealmOptions=
+  // Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. This reference overrides any configured SSLContextParameters at the component level. See Using the JSSE Configuration Utility. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level.
+  // camel.sink.endpoint.sslContextParameters=
+  // Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail during starting and cause the route to fail being started. By deferring this startup to be lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. Beware that when the first message is processed then creating and starting the producer may take a little time and prolong the total processing time of the processing.
+  // camel.component.ahc-ws.lazyStartProducer=
+  // Whether to allow java serialization when a request uses context-type=application\/x-java-serialized-object This is by default turned off. If you enable this then be aware that Java will deserialize the incoming data from the request to Java and that can be a potential security risk.
+  // camel.component.ahc-ws.allowJavaSerializedObject=
+  // Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching type, which then gets configured on the component. This can be used for automatic configuring JDBC data sources, JMS connection factories, AWS Clients, etc.
+  // camel.component.ahc-ws.autowiredEnabled=
+  // To use a custom AhcBinding which allows to control how to bind between AHC and Camel.
+  // camel.component.ahc-ws.binding=
+  // To use a custom AsyncHttpClient
+  // camel.component.ahc-ws.client=
+  // To configure the AsyncHttpClient to use a custom com.ning.http.client.AsyncHttpClientConfig instance.
+  // camel.component.ahc-ws.clientConfig=
+  // To use a custom org.apache.camel.spi.HeaderFilterStrategy to filter header to and from Camel message.
+  // camel.component.ahc-ws.headerFilterStrategy=
+  // Reference to a org.apache.camel.support.jsse.SSLContextParameters in the Registry. Note that configuring this option will override any SSL\/TLS configuration options provided through the clientConfig option at the endpoint or component level.
+  // camel.component.ahc-ws.sslContextParameters=
+  // Enable usage of global SSL context parameters.
+  // camel.component.ahc-ws.useGlobalSslContextParameters=
+}

modules/web-map/build.gradle.kts

@@ -59,7 +59,7 @@ kotlin {
           )
         )
 
-        implementation(npm("kafkajs", "1.15.0"))
+//        implementation(npm("kafkajs", "1.15.0"))
       }
 
       val webDir = file("src/main/web")