From 4329651c863f7e395c700dc0ae7b07478da6a657 Mon Sep 17 00:00:00 2001 From: Adam <897017+asemy@users.noreply.github.com> Date: Sun, 24 Oct 2021 14:55:04 +0200 Subject: [PATCH] tidy up shared config, start setting up kafka processor --- .gitignore | 3 + .../archetype/kotlin-jvm.gradle.kts | 30 +++++++- gradle/libs.versions.toml | 15 +++- .../build.gradle.kts | 20 +++++ .../processor/FactorioEventLogConsumer.kt | 7 ++ .../processor/FactorioEventsTopology.kt | 20 +++++ .../processor/StreamsExceptionHandler.kt | 11 +++ .../processor/config/ApplicationProperties.kt | 68 +++++++++++++++++ .../factorioevents/processor/config/Lenses.kt | 38 ++++++++++ .../processor/config/PortVal.kt | 30 ++++++++ .../processor/config/environmentExt.kt | 73 +++++++++++++++++++ .../adamko/factorioevents/processor/main.kt | 19 +++++ modules/kt-rcon/build.gradle.kts | 2 +- 13 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventLogConsumer.kt create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventsTopology.kt create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/StreamsExceptionHandler.kt create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/ApplicationProperties.kt create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/Lenses.kt create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/PortVal.kt create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/environmentExt.kt create mode 100644 modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/main.kt diff --git a/.gitignore b/.gitignore index acd20e6..57dfac2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ +# ignore any files that start with '.secret' +.secret.* + .idea node_modules diff --git a/buildSrc/src/main/kotlin/dev/adamko/factoriowebmap/archetype/kotlin-jvm.gradle.kts b/buildSrc/src/main/kotlin/dev/adamko/factoriowebmap/archetype/kotlin-jvm.gradle.kts index ef894b0..7e3553c 100644 --- a/buildSrc/src/main/kotlin/dev/adamko/factoriowebmap/archetype/kotlin-jvm.gradle.kts +++ b/buildSrc/src/main/kotlin/dev/adamko/factoriowebmap/archetype/kotlin-jvm.gradle.kts @@ -9,12 +9,36 @@ plugins { kotlin { jvmToolchain { - (this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(11)) // "8" + (this as JavaToolchainSpec).languageVersion.set(JavaLanguageVersion.of(11)) } } dependencies { + + // + implementation(platform("org.jetbrains.kotlin:kotlin-bom")) + implementation(kotlin("stdlib")) + + // + + // + + val junitVersion = "5.8.1" + testImplementation(enforcedPlatform("org.junit:junit-bom:$junitVersion")) + testImplementation("org.junit.jupiter:junit-jupiter-api") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") + + val kotestVersion = "4.6.3" + testImplementation(enforcedPlatform("io.kotest:kotest-bom:$kotestVersion")) + testImplementation("io.kotest:kotest-runner-junit5-jvm:$kotestVersion") + testImplementation("io.kotest:kotest-assertions-core-jvm:$kotestVersion") + testImplementation("io.kotest:kotest-property-jvm:$kotestVersion") + testImplementation("io.kotest:kotest-assertions-json:$kotestVersion") + testImplementation("io.mockk:mockk:1.12.0") + // + } tasks.withType().configureEach { @@ -24,4 +48,8 @@ tasks.withType().configureEach { "-Xopt-in=kotlin.time.ExperimentalTime", "-Xopt-in=kotlinx.coroutines.ExperimentalCoroutinesApi", ) +} + +tasks.compileTestKotlin { + kotlinOptions.freeCompilerArgs += "-Xopt-in=io.kotest.common.ExperimentalKotest" } \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b9724e7..cadbdd3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -9,13 +9,14 @@ jvm = "11" kotlin = "1.5.31" ## libs ## -http4k = "4.13.1.0" +http4k = "4.16.1.0" slf4j = "1.7.30" log4j2 = "2.14.1" qosLogback = "1.2.5" +kafka = "3.0.0" ## test ## -junit = "5.8.0" +junit = "5.8.1" kotest = "4.6.3" @@ -59,6 +60,7 @@ http4k-cloudnative = { group = "org.http4k", name = "http4k-cloudnative" http4k-format-json = { group = "org.http4k", name = "http4k-format-jackson" } http4k-format-xml = { group = "org.http4k", name = "http4k-format-jackson-xml" } http4k-format-yaml = { group = "org.http4k", name = "http4k-format-jackson-yaml" } +http4k-graphql = { group = "org.http4k", name = "http4k-graphql" } ## logging ## slf4j-api = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" } @@ -73,6 +75,14 @@ gcp-pubsub = { group = "com.google.cloud", name = "google-cloud-pubsub" gcp-speech = { group = "com.google.cloud", name = "google-cloud-speech" } +## kafka ## + +kafka-streams = { group = "org.apache.kafka", name = "kafka-streams", version.ref = "kafka" } + +# kafka helpers +kotka = { group = "com.github.blueanvil", name = "kotka", version = "1.1.1" } + + ############# ### test ### ############# @@ -86,6 +96,7 @@ junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", junit-launcher = { group = "org.junit.platform", name = "junit-platform-launcher", version = "1.7.2" } ## kotest ## +kotest-bom = { group = "io.kotest", name = "kotest-bom", version.ref = "kotest" } kotest-core = { group = "io.kotest", name = "kotest-assertions-core-jvm", version.ref = "kotest" } kotest-datatest = { group = "io.kotest", name = "kotest-framework-datatest", version.ref = "kotest" } kotest-runner = { group = "io.kotest", name = "kotest-runner-junit5-jvm", version.ref = "kotest" } diff --git a/modules/factorio-events-processor/build.gradle.kts b/modules/factorio-events-processor/build.gradle.kts index e5eff9b..9a44560 100644 --- a/modules/factorio-events-processor/build.gradle.kts +++ b/modules/factorio-events-processor/build.gradle.kts @@ -1,5 +1,7 @@ + plugins { id("dev.adamko.factoriowebmap.archetype.kotlin-jvm") + application } description = @@ -12,3 +14,21 @@ description = val projectId: String by project.extra val buildDir: Directory = layout.buildDirectory.dir(projectId).get() +@Suppress("UnstableApiUsage") +dependencies { + + implementation("org.apache.kafka:kafka-streams:3.0.0") + + implementation(platform(libs.http4k.bom)) + implementation(libs.http4k.core) + implementation(libs.http4k.cloudnative) + implementation(libs.http4k.format.json) + implementation(libs.http4k.format.yaml) + + implementation(libs.bundles.logging) + +} + +application { + mainClass.set("dev.adamko.factorioevents.processor.mainKt") +} diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventLogConsumer.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventLogConsumer.kt new file mode 100644 index 0000000..091f084 --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventLogConsumer.kt @@ -0,0 +1,7 @@ +package dev.adamko.factorioevents.processor + +/** + * Process the raw messages from a server. + */ +class FactorioEventLogConsumer { +} \ No newline at end of file diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventsTopology.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventsTopology.kt new file mode 100644 index 0000000..83924d9 --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/FactorioEventsTopology.kt @@ -0,0 +1,20 @@ +package dev.adamko.factorioevents.processor + +import java.util.Properties +import org.apache.kafka.streams.KafkaStreams +import org.apache.kafka.streams.StreamsBuilder + +class FactorioEventsTopology { + + fun build() { + + val topology = StreamsBuilder().build() + + + val properties = Properties() + + val kafkaStreams = KafkaStreams(topology, properties) + + } + +} \ No newline at end of file diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/StreamsExceptionHandler.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/StreamsExceptionHandler.kt new file mode 100644 index 0000000..bf05679 --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/StreamsExceptionHandler.kt @@ -0,0 +1,11 @@ +package dev.adamko.factorioevents.processor + +import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler +import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse + +class StreamsExceptionHandler : StreamsUncaughtExceptionHandler { + override fun handle(exception: Throwable): StreamThreadExceptionResponse { + println("${exception.message} - ${exception.stackTrace}") + return StreamThreadExceptionResponse.REPLACE_THREAD + } +} diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/ApplicationProperties.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/ApplicationProperties.kt new file mode 100644 index 0000000..86232c8 --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/ApplicationProperties.kt @@ -0,0 +1,68 @@ +package dev.adamko.factorioevents.processor.config + +import com.fasterxml.jackson.databind.JsonNode +import dev.adamko.factorioevents.processor.config.PortVal.Companion.portVal +import java.io.StringReader +import java.time.Clock +import java.util.Properties +import org.http4k.cloudnative.env.Environment +import org.http4k.cloudnative.env.EnvironmentKey +import org.http4k.core.Credentials +import org.http4k.core.Uri +import org.http4k.format.Jackson +import org.http4k.format.Json +import org.http4k.lens.boolean +import org.http4k.lens.map +import org.http4k.lens.nonEmptyString +import org.http4k.lens.uri + +/** + * Load properties, in descending priority order, from + * + * 1. Environment variables + * 2. This app's env file (as defined by the environment variable [Lenses.APPLICATION_PROPERTIES_FILE_PATH]) + * # 3. A resources-dir yaml file + * + */ +class ApplicationProperties( + vararg environments: Environment = arrayOf( + Environment.env(), + Environment.fromEnvVarPath(), + Environment.fromResourcesFile(".secret.config.yml") + ) +) { + private val appProps: Environment = + environments.reduce { acc, environment -> acc.overrides(environment) } + + fun clock(): Clock = Clock.systemUTC() + + // + + /** + * The port of the server + * + * Set by GCP ([docs](https://cloud.google.com/run/docs/reference/container-contract#env-vars)). + */ + val serverPort: PortVal + get() = EnvironmentKey.portVal().defaulted("PORT", PortVal.DEFAULT)[appProps] + + // + + + // + + val kafkaConfig: Properties + get() = EnvironmentKey + .nonEmptyString() + .map { + it.reader().use { sr -> + Properties().also { p -> + p.load(sr) + } + } + } + .required("kafka.config")[appProps] + + // + +} diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/Lenses.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/Lenses.kt new file mode 100644 index 0000000..26e5109 --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/Lenses.kt @@ -0,0 +1,38 @@ +package dev.adamko.factorioevents.processor.config + +import java.io.File +import org.http4k.cloudnative.env.Environment +import org.http4k.cloudnative.env.EnvironmentKey +import org.http4k.core.Credentials +import org.http4k.lens.BiDiLens +import org.http4k.lens.BiDiMapping + + +internal object Lenses { + + /** The location of this app's env file. */ + val APPLICATION_PROPERTIES_FILE_PATH: BiDiLens = + EnvironmentKey + .map( + { path: String -> File(path) }, + { file: File -> file.canonicalPath }, + ) + .optional("APPLICATION_PROPERTIES_FILE_PATH") + + /** Bidirectional [String] and [Credentials] mapper. */ + val credentialsLens: BiDiMapping = run { + val separator = ":" + BiDiMapping( + // from String to Credentials + { property -> + property.split(separator, limit = 2) + .takeIf { it.size == 2 } + ?.let { (user, pass) -> Credentials(user, pass) } + ?: throw IllegalArgumentException("Credentials are invalid") + }, + // from Credentials to String + { it.user + separator + it.password } + ) + } + +} diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/PortVal.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/PortVal.kt new file mode 100644 index 0000000..498dc77 --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/PortVal.kt @@ -0,0 +1,30 @@ +package dev.adamko.factorioevents.processor.config + +import org.http4k.cloudnative.env.Port +import org.http4k.lens.BiDiLensSpec +import org.http4k.lens.StringBiDiMappings.int +import org.http4k.lens.map + +/** + * Improved (slightly) version of http4k's [Port]. + */ +@JvmInline +value class PortVal(val value: Int) { + + init { + require(value in VALID_RANGE) { + "Out of range. Port: '$value', range: $VALID_RANGE" + } + } + + operator fun invoke(): Int = value + + companion object { + val DEFAULT = PortVal(8080) + val VALID_RANGE: IntRange + get() = (1..65535) + + fun BiDiLensSpec.portVal(): BiDiLensSpec = + map(int().map(::PortVal, PortVal::value)) + } +} diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/environmentExt.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/environmentExt.kt new file mode 100644 index 0000000..d0a837f --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/config/environmentExt.kt @@ -0,0 +1,73 @@ +package dev.adamko.factorioevents.processor.config + +import java.io.File +import java.io.FileNotFoundException +import org.http4k.cloudnative.env.Environment +import org.http4k.cloudnative.env.MapEnvironment +import org.http4k.cloudnative.env.fromYaml +import org.http4k.lens.Lens + + +/** + * Load properties from either a `.yaml` or `.env` file. + * + * (An improved version of [Environment.fromResource]) + */ +internal fun Environment.Companion.fromResourcesFile( + resource: String, + default: Environment = EMPTY +) = + try { + val file = + this::class.java + .getResource("/${resource.removePrefix("/")}") + ?.file + ?.let { File(it) } + fromFile(file, default) + } catch (e: FileNotFoundException) { + default + } + +/** + * Try fetching a properties file the location provided by the + * [`JVM`][jvmProperties] or [`System`][env] environment variable + * [Lenses.APPLICATION_PROPERTIES_FILE_PATH], if it exists. + * + * If it does not exist, return [default]. + */ +internal fun Environment.Companion.fromEnvVarPath( + location: Lens = Lenses.APPLICATION_PROPERTIES_FILE_PATH, + default: Environment = EMPTY, +): Environment { + val file = (jvmProperties() overrides env())[location] + + return fromFile(file, default) +} + +/** + * Load properties from either a `.yaml` or `.env` file. + * + * (An improved version of [Environment.from]) + */ +internal fun Environment.Companion.fromFile( + file: File?, + default: Environment = EMPTY, +): Environment { + return file + ?.takeIf { it.isFile } + .let { + when (it?.extension) { + "yml", "yaml" -> fromYaml(it) + "env" -> from(it) + else -> default + } + } +} + +/** The same as [Environment.ENV], but refreshes on each invocation (which is useful for testing). */ +internal fun Environment.Companion.env(): Environment = + MapEnvironment.from(System.getenv().toProperties()) + +/** The same as [Environment.JVM_PROPERTIES], but refreshes on each invocation (which is useful for testing). */ +internal fun Environment.Companion.jvmProperties(): Environment = + MapEnvironment.from(System.getProperties()) diff --git a/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/main.kt b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/main.kt new file mode 100644 index 0000000..f0df841 --- /dev/null +++ b/modules/factorio-events-processor/src/main/kotlin/dev/adamko/factorioevents/processor/main.kt @@ -0,0 +1,19 @@ +package dev.adamko.factorioevents.processor + +import dev.adamko.factorioevents.processor.config.ApplicationProperties +import org.apache.kafka.streams.KafkaStreams +import org.apache.kafka.streams.StreamsBuilder + +fun main() { + + val appProps = ApplicationProperties() + + val topology = StreamsBuilder().build() + + val properties = appProps.kafkaConfig + + val streams = KafkaStreams(topology, properties) + + streams.setUncaughtExceptionHandler(StreamsExceptionHandler()) + +} \ No newline at end of file diff --git a/modules/kt-rcon/build.gradle.kts b/modules/kt-rcon/build.gradle.kts index dbd36d2..cab5d06 100644 --- a/modules/kt-rcon/build.gradle.kts +++ b/modules/kt-rcon/build.gradle.kts @@ -8,7 +8,7 @@ plugins { dependencies { - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2") + implementation(libs.kotlinx.coroutines) implementation("io.ktor:ktor-network:1.6.4")