From 995ea750be076b6688048224ce7e78c3a3176e47 Mon Sep 17 00:00:00 2001 From: Adam <897017+asemy@users.noreply.github.com> Date: Sat, 23 Oct 2021 23:21:31 +0200 Subject: [PATCH] rm old kafka-pipe experiments --- modules/factorio-events-kafka-pipe/.env | 1 + .../build.gradle.kts | 145 +----------------- .../docker-compose.yml | 1 + .../kafka-pipe-jar.Dockerfile | 12 -- .../kafka-pipe.Dockerfile | 22 +-- .../pipe-logs-to-kafka.sh | 50 ------ modules/factorio-events-kafka-pipe/readme.md | 13 +- .../src/main/kotlin/fekp.kt | 17 -- 8 files changed, 15 insertions(+), 246 deletions(-) create mode 100644 modules/factorio-events-kafka-pipe/.env delete mode 100644 modules/factorio-events-kafka-pipe/kafka-pipe-jar.Dockerfile delete mode 100644 modules/factorio-events-kafka-pipe/pipe-logs-to-kafka.sh delete mode 100644 modules/factorio-events-kafka-pipe/src/main/kotlin/fekp.kt diff --git a/modules/factorio-events-kafka-pipe/.env b/modules/factorio-events-kafka-pipe/.env new file mode 100644 index 0000000..d3d9e29 --- /dev/null +++ b/modules/factorio-events-kafka-pipe/.env @@ -0,0 +1 @@ +COMPOSE_PROJECT_NAME=factorio-events-kafka-pipe diff --git a/modules/factorio-events-kafka-pipe/build.gradle.kts b/modules/factorio-events-kafka-pipe/build.gradle.kts index fc935c8..e36b82f 100644 --- a/modules/factorio-events-kafka-pipe/build.gradle.kts +++ b/modules/factorio-events-kafka-pipe/build.gradle.kts @@ -1,11 +1,7 @@ import org.jetbrains.kotlin.util.parseSpaceSeparatedArgs plugins { - id("dev.adamko.factoriowebmap.archetype.kotlin-jvm") - - application - -// id("com.github.johnrengelman.shadow") version "7.1.0" + id("dev.adamko.factoriowebmap.archetype.base") } description = @@ -14,142 +10,3 @@ description = """.trimIndent() val projectId: String by project.extra - -dependencies { - implementation(libs.bundles.logging) - implementation("eu.jrie.jetbrains:kotlin-shell-core:0.2.1") -} - -application { - mainClass.set("factorioEventsKafkaPipeKt") -} - -java { - withSourcesJar() -} - -tasks.jar { - manifest { - attributes( - "Implementation-Title" to project.name, - "Implementation-Version" to project.version, - "Main-Class" to application.mainClass, - ) - } - archiveBaseName.set(project.name) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - from( - configurations.runtimeClasspath.map { config -> - config.map { - logger.trace("Adding runtimeClasspath file to jar - $it") - when (it.isDirectory) { - true -> it - false -> zipTree(it) - } - } - } - ) - -} - -tasks.create("dockerBuildPdl2k") { - group = projectId - - dependsOn(tasks.jar) - - val jar = tasks.jar.get().archiveFile.get() - logger.lifecycle("building docker with jar $jar") - inputs.file(jar) - - commandLine( - parseSpaceSeparatedArgs( - "docker-compose build pdl2k --build-arg JAR_LOCATION=${jar.asFile.canonicalPath}" - ) - ) - -// inputs(tasks.jar.get().archiveFile.get()) - -// commandLine( -// "docker", -// "build", -// "--build-arg", -// "JAR_FILE=", -// "-t dev-adamko/pdl2k" -// ) -} - -//tasks.jar.get().archiveFile.get().asFile.canonicalPath - -tasks.create("runInDocker") { - group = projectId -// docker run -v `pwd`:/mnt java:8 java -jar /mnt/myapp.jar - - dependsOn(tasks.jar) - val jarFile = tasks.jar.get().archiveFile.get().asFile - inputs.file(jarFile) - - commandLine( - listOf( - "docker", - "run", - "--rm", - "--name", project.name, -// """--workdir="/app" """, - "-v", ".//${jarFile.toRelativeString(project.projectDir).replace("\\", "//")}:/mnt/", - "openjdk:11.0.6-jre-buster", - "ls", "-la", "/mnt/*", -// "java", "-jar", "/mnt/${jarFile.name}", - ) - ) - - doFirst { - logger.lifecycle("Executing [$commandLine]") - } -} - -//afterEvaluate { -// tasks { -// create("jar2", Jar::class).apply { -// dependsOn("jvmMainClasses", "jsJar") -// group = "jar" -// manifest { -// attributes( -// mapOf( -// "Implementation-Title" to rootProject.name, -// "Implementation-Version" to rootProject.version, -// "Timestamp" to System.currentTimeMillis(), -// "Main-Class" to application.mainClass -// ) -// ) -// } -// -// from( -// configurations -// ) -// -// val dependencies = -// configurations["jvmRuntimeClasspath"] -// .filter { it.name.endsWith(".jar") } + -// project.tasks["jvmJar"].outputs.files -// dependencies.forEach { from(zipTree(it)) } -// into("/lib") -// } -// } -//} -//val fatJar = task("fatJar", type = Jar::class) { -// baseName = "safescape-lib-${project.name}" -// // manifest Main-Class attribute is optional. -// // (Used only to provide default main class for executable jar) -// from(configurations.runtimeClasspath.map({ if (it.isDirectory) it else zipTree(it) })) -// with(tasks["jar"] as CopySpec) -//} -// -//tasks { -// "build" { -// dependsOn(fatJar) -// } -//} -//from(configurations.runtimeClasspath) { -// into("lib") -//} \ No newline at end of file diff --git a/modules/factorio-events-kafka-pipe/docker-compose.yml b/modules/factorio-events-kafka-pipe/docker-compose.yml index b97d5e8..4d42dbd 100644 --- a/modules/factorio-events-kafka-pipe/docker-compose.yml +++ b/modules/factorio-events-kafka-pipe/docker-compose.yml @@ -6,6 +6,7 @@ services: container_name: "factorio-events-kafka-pipe" build: dockerfile: "kafka-pipe.Dockerfile" + privileged: true networks: - "factorio-events-nw" environment: diff --git a/modules/factorio-events-kafka-pipe/kafka-pipe-jar.Dockerfile b/modules/factorio-events-kafka-pipe/kafka-pipe-jar.Dockerfile deleted file mode 100644 index 16ded19..0000000 --- a/modules/factorio-events-kafka-pipe/kafka-pipe-jar.Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -ARG JAR_LOCATION - -FROM openjdk:11.0.6-jre-buster -ARG JAR_LOCATION - -RUN apt-get update && apt-get install -y \ - curl \ - && rm -rf /var/lib/apt/lists/* - -COPY JAR_LOCATION /app.jar - -ENTRYPOINT ["java","-jar","/app.jar"] diff --git a/modules/factorio-events-kafka-pipe/kafka-pipe.Dockerfile b/modules/factorio-events-kafka-pipe/kafka-pipe.Dockerfile index 9fc1c79..ce5b414 100644 --- a/modules/factorio-events-kafka-pipe/kafka-pipe.Dockerfile +++ b/modules/factorio-events-kafka-pipe/kafka-pipe.Dockerfile @@ -1,31 +1,15 @@ - -#FROM debian:buster-slim FROM docker:dind-rootless ENV FACTORIO_SERVER_CONTAINER_NAME="factorio-server" -ENV KAFKA_HOST="kafka-kraft" +ENV KAFKA_HOST="kafka" -#RUN apt-get update && apt-get install -y --no-install-recommends \ -# curl \ -# docker \ -# kafkacat - -# root is required for following Docker logs +# root is required for installing dependencies USER root - RUN apk add --update \ curl \ kafkacat -#USER rootless - -# \ -# && rm -rf /var/lib/apt/lists/* +USER rootless COPY kafka-pipe.sh kafka-pipe.sh -#RUN useradd --user-group --system --create-home --no-log-init the_engineer -#USER the_engineer - ENTRYPOINT [ "/bin/sh", "/kafka-pipe.sh" ] -#ENTRYPOINT /bin/bash -#CMD exec /kafka-pipe.sh diff --git a/modules/factorio-events-kafka-pipe/pipe-logs-to-kafka.sh b/modules/factorio-events-kafka-pipe/pipe-logs-to-kafka.sh deleted file mode 100644 index d89e815..0000000 --- a/modules/factorio-events-kafka-pipe/pipe-logs-to-kafka.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/zsh - -# stream logs from 'factorio' container -# follow the logs, and -# combine stdout and stderr -# only match lines that start with 'FactorioEvent: ', -# and only return the stuff *after* 'FactorioEvent: ' -docker logs --tail 0 -f factorio-server 2>&1 | - sed -n -e 's/^FactorioEvent: //p' | - kafkacat -P -b localhost -t factorio-server-log - -# listen to events -# kafkacat -C -b localhost -t factorio-server-log | jq - -# curl -sS --unix-socket /var/run/docker.sock "http://localhost/v1.41/events?filters={"service":"factorio-server"}" --output - | jq - -curl --no-buffer -sS \ - --unix-socket /var/run/docker.sock \ - "http://localhost/v1.41/containers/factorio-server/logs?stdout=true&follow=true&tail=0" | - tail -c+9 -# --output - - -curl \ - -sS \ - --unix-socket /var/run/docker.sock \ - -X POST \ - --insecure "http://localhost/v1.41/containers/factorio-server/attach?logs=true&stream=true&stdout=true&follow=true" | - tail -c+9 - -# sed 's/^........//' - - -# https://stackoverflow.com/questions/42873285/curl-retry-mechanism -# --retry-all-errors - -# --header "Sec-WebSocket-Key: SGVsbG8sIHdvcmxkIQ==" \ -# --header "Sec-WebSocket-Version: 13" \ -# --header "Host: localhost:80" \ -# --header "Origin: http://localhost:80" \ - -curl --include \ - --unix-socket /var/run/docker.sock \ - --no-buffer \ - --header "Connection: Upgrade" \ - --header "Upgrade: websocket" \ - --header "Host: localhost:80" \ - --header "Origin: http://localhost:80" \ - --header "Sec-WebSocket-Version: 13" \ - --header "Sec-WebSocket-Key: SGVsbG8sIHdvcmxkIQ==" \ - "http://localhost/v1.41/containers/factorio-server/attach/ws?logs=true&stdout=true" diff --git a/modules/factorio-events-kafka-pipe/readme.md b/modules/factorio-events-kafka-pipe/readme.md index 5d9977d..2fc4129 100644 --- a/modules/factorio-events-kafka-pipe/readme.md +++ b/modules/factorio-events-kafka-pipe/readme.md @@ -1,8 +1,6 @@ - ### Setup notes -Kafka must be connectable from both `localhost` (for cli) and `kafa` (the docker container name). - +Kafka must be connectable from both `localhost` (for cli) and `kafa` (the docker container name). ### Useful commands @@ -43,5 +41,12 @@ curl --no-buffer -sS --unix-socket /var/run/docker.sock \ # get logs from Kafka kafkacat -u -C -b localhost -t factorio-server-log | jq --unbuffered -``` +# stream logs, and try trimming first 8 bytes (the header) but it doesn't work +curl \ + -sS \ + --unix-socket /var/run/docker.sock \ + -X POST \ + --insecure "http://localhost/v1.41/containers/factorio-server/attach?logs=true&stream=true&stdout=true&follow=true" | + tail -c+9 +``` \ No newline at end of file diff --git a/modules/factorio-events-kafka-pipe/src/main/kotlin/fekp.kt b/modules/factorio-events-kafka-pipe/src/main/kotlin/fekp.kt deleted file mode 100644 index be87043..0000000 --- a/modules/factorio-events-kafka-pipe/src/main/kotlin/fekp.kt +++ /dev/null @@ -1,17 +0,0 @@ -import eu.jrie.jetbrains.kotlinshell.shell.shell - -fun main() { - shell { - - "echo hello world"() - -// """ -// curl \ -// --no-buffer \ -// -sS \ -// --unix-socket /var/run/docker.sock \ -// "http://localhost/v1.41/containers/factorio-server/logs?stdout=true&follow=true&tail=0" -// """.trimIndent() - - } -} \ No newline at end of file