project re-organisation, and rename

modules/infra-kafka-pipe/build.gradle.kts

@@ -0,0 +1,6 @@
+plugins {
+  id("dev.adamko.factoriowebmap.archetype.process-runner")
+}
+
+description = "Send events from a Factorio server to a Kafka topic "
+

modules/infra-kafka-pipe/src/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=factorio-events-kafka-pipe

modules/infra-kafka-pipe/src/docker-compose.yml

@@ -0,0 +1,25 @@
+version: "3.9"
+
+services:
+  factorio-events-kafka-pipe:
+    image: openjdk:11-jre-buster
+    container_name: "factorio-events-kafka-pipe"
+    build:
+      dockerfile: "kafka-pipe.Dockerfile"
+    restart: "unless-stopped"
+    privileged: true
+    networks:
+      - "factorio-events-nw"
+    environment:
+      FACTORIO_SERVER_CONTAINER_NAME: "factorio-server"
+      KAFKA_HOST: "kafka:29092"
+    volumes:
+      # Windows (requires privileged/root)
+      - "//var/run/docker.sock:/var/run/docker.sock"
+      # Unix?
+      # - "/var/run/docker.sock:/var/run/docker.sock"
+
+networks:
+  factorio-events-nw:
+    name: "factorio-events-nw"
+    external: true

modules/infra-kafka-pipe/src/kafka-pipe.Dockerfile

@@ -0,0 +1,16 @@
+FROM docker:dind-rootless
+
+ENV FACTORIO_SERVER_CONTAINER_NAME="factorio-server"
+ENV KAFKA_HOST="kafka"
+
+# root is required for installing dependencies and
+# using the docker socket (listening to logs)
+USER root
+RUN apk add --update \
+    curl \
+    kafkacat
+#USER rootless
+
+COPY kafka-pipe.sh kafka-pipe.sh
+
+ENTRYPOINT [ "/bin/sh", "/kafka-pipe.sh" ]

modules/infra-kafka-pipe/src/kafka-pipe.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/bash
+
+# Pipe Docker logs from a Factorio server to a Kafka topic
+
+echo "Piping logs from $FACTORIO_SERVER_CONTAINER_NAME to $KAFKA_HOST"
+
+trap endProcess SIGINT
+endProcess() {
+  echo "Exiting Factorio Events Kafka Pipe"
+  exit
+}
+
+# some random ID, to use as a message key
+INSTANCE_ID=$(cat /proc/sys/kernel/random/uuid)
+
+while :; do
+  docker logs --tail 0 -f "$FACTORIO_SERVER_CONTAINER_NAME" |
+    sed -n -e 's/^FactorioEvent: //p' |
+    kafkacat -P -T -b "$KAFKA_HOST" -t factorio-server-log -k "$INSTANCE_ID"
+
+  echo "Error - retrying in 10 seconds"
+  sleep 10
+
+done

modules/infra-kafka-pipe/src/readme.md

@@ -0,0 +1,52 @@
+### Setup notes
+
+Kafka must be connectable from both `localhost` (for cli) and `kafa` (the docker container name).
+
+### Useful commands
+
+#### Powershell
+
+```shell
+# might help with mounting the unix socket on Windows? 
+$Env:COMPOSE_CONVERT_WINDOWS_PATHS=1 
+
+# full rebuild
+docker-compose down; docker-compose build; docker-compose up -d; docker-compose logs -f
+
+```
+
+#### zsh
+
+```shell
+# listen to events from factorio-server
+docker logs --tail 0 -f factorio-server | sed -n -e 's/^FactorioEvent: //p' 
+
+# tail logs with docker api (except it doesn't work)
+curl -vvv \ 
+  --unix-socket /var/run/docker.sock \
+  --no-buffer \
+  --header "Connection: Upgrade" \
+  --header "Upgrade: websocket" \
+  --header "Host: localhost:80" \
+  --header "Origin: http://localhost:80" \
+  --header "Sec-WebSocket-Version: 13" \
+  --header "Sec-WebSocket-Key: SGVsbG8sIHdvcmxkIQ==" \
+  "http://localhost:80/v1.41/containers/factorio-server/attach/ws?stream=true&logs=true&stdout=true"
+
+# tail logs 
+curl --no-buffer -sS --unix-socket /var/run/docker.sock \
+    --get --data-urlencode 'filters={"container":["my-service"]}' \
+    "http://localhost/v1.41/events" | jq --unbuffered
+
+# get logs from Kafka
+kafkacat -u -C -b localhost -t factorio-server-log | jq --unbuffered   
+
+# stream logs, and try trimming first 8 bytes (the header) but it doesn't work
+curl \
+  -sS \
+  --unix-socket /var/run/docker.sock \
+  -X POST \
+  --insecure "http://localhost/v1.41/containers/factorio-server/attach?logs=true&stream=true&stdout=true&follow=true" |
+  tail -c+9
+
+```