From 5bb8cc789cdbcc407d7f39166ce01d8c4a3fb32a Mon Sep 17 00:00:00 2001
From: Adam <897017+aSemy@users.noreply.github.com>
Date: Sun, 30 Oct 2022 17:21:32 +0100
Subject: [PATCH] add/update docs
---
README.md | 33 +++--
docs/guide.md | 52 +++++++
.../config/kafka-server.properties | 130 ++++++++++++++++
docs/kafkatorio-platform/docker-compose.yml | 140 ++++++++++++++++++
docs/kafkatorio-platform/example.env | 10 ++
.../example.secret.kafkatorio-config.yml | 10 ++
gradle/libs.versions.toml | 2 +-
7 files changed, 361 insertions(+), 16 deletions(-)
create mode 100644 docs/guide.md
create mode 100644 docs/kafkatorio-platform/config/kafka-server.properties
create mode 100644 docs/kafkatorio-platform/docker-compose.yml
create mode 100644 docs/kafkatorio-platform/example.env
create mode 100644 docs/kafkatorio-platform/example.secret.kafkatorio-config.yml
diff --git a/README.md b/README.md
index 54c20e7..e0c3d07 100644
--- a/README.md
+++ b/README.md
@@ -15,21 +15,8 @@ inter-server communication.
### Overview
-```mermaid
-sequenceDiagram
-
-Factorio multiplayer server->>events-mod: in-game events
-Note over events-mod: converts events to packets
-events-mod->>logs processor: packets logged to console
-logs processor->>Kafka: publishes packets
(Requires API key!)
-
-Kafka->>events-server:
-
-Note over events-server: Packets are processed
-
-events-server->>web-map: Web map tiles (REST API)
-events-server-)web-map: WebSocket updates
(e.g. player movement)
-```
+This is a brief overview of how Kafkatorio gathers data, processes it, and uses it to create the
+live web-map.
1. The Kafkatorio game mod, [`events-mod`](./modules/events-mod), collects event data, de-bouncing
and grouping events, and converts them to JSON packets
@@ -47,6 +34,22 @@ events-server-)web-map: WebSocket updates
(e.g. player movement)
5. The `web-map` uses Leaflet/JS to display the map, using the tiles and WebSocket messages from
`events-server`
+```mermaid
+sequenceDiagram
+
+Factorio multiplayer server->>events-mod: in-game events
+Note over events-mod: converts events to packets
+events-mod->>logs processor: packets logged to console
+logs processor->>Kafka: publishes packets
(Requires API key!)
+
+Kafka->>events-server:
+
+Note over events-server: Packets are processed
+
+events-server->>web-map: Web map tiles (REST API)
+events-server-)web-map: WebSocket updates
(e.g. player movement)
+```
+
## License
While this project is under development, [no license](https://choosealicense.com/no-permission/)
diff --git a/docs/guide.md b/docs/guide.md
new file mode 100644
index 0000000..8fd2a94
--- /dev/null
+++ b/docs/guide.md
@@ -0,0 +1,52 @@
+# Running Kafkatorio
+
+### Building the project
+
+#### Requirements
+
+* Docker
+* Docker Compose
+* Java 11
+
+Kafkatorio uses [Gradle](https://docs.gradle.org/current/userguide/getting_started.html) to
+build the project, and Docker Compose to create images.
+
+* `./gradlew build` - build and test the project
+* `./gradlew runKafkatorio` - run locally (if Docker is running, a Kafka instance will start)
+* `./gradlew dockerComposeBuild` - build, test, and create Kafkatorio Docker images
+
+### Server set up
+
+> See [./kafkatorio-platform/docker-compose.yml](kafkatorio-platform/docker-compose.yml) for a
+> complete example
+
+1. Copy the [kafkatorio-platform](./kafkatorio-platform) directory to your computer
+2. Rename
+ [`example.secret.kafkatorio-config.yml`](./kafkatorio-platform/example.secret.kafkatorio-config.yml)
+ to `.secret.kafkatorio-config.yml`
+
+ In it, define both
+ * a JWT secret password (a long alphanumeric string is best, you won't need to type this
+ password manually)
+ * a Factorio server, with a short-ID and a description.
+3. Rename [`example.env`](./kafkatorio-platform/example.env) to `.env`
+4. Create a JWT for your Factorio server, by going to https://jwt.io/
+
+ * In 'PAYLOAD: DATA', set `sub` to your Factorio server's short ID.
+ ```json
+ {
+ "sub": "",
+ "iat": ...
+ }
+ ```
+ (`iat`, 'issued-at-time', should update automatically to be the current time.)
+ * Replace `your-256-bit-secret` with your JWT secret password
+
+ Copy the 'Encoded' token (it should start with `ey`).
+
+ In the `.env` file, set `KAFKATORIO_TOKEN` to equal the token.
+5. In `kafkatorio-platform`, start the docker containers
+
+ ```terminal
+ docker-compose up -d; docker-compose logs -f
+ ```
diff --git a/docs/kafkatorio-platform/config/kafka-server.properties b/docs/kafkatorio-platform/config/kafka-server.properties
new file mode 100644
index 0000000..dd3de1f
--- /dev/null
+++ b/docs/kafkatorio-platform/config/kafka-server.properties
@@ -0,0 +1,130 @@
+# This configuration file is intended for use in KRaft mode, where
+# Apache ZooKeeper is not present. See config/kraft/README.md for details.
+
+############################# Server Basics #############################
+
+# The role of this server. Setting this puts us in KRaft mode
+process.roles=broker,controller
+
+# The node id associated with this instance's roles
+node.id=1
+
+# The connect string for the controller quorum
+controller.quorum.voters=1@localhost:9093
+
+############################# Socket Server Settings #############################
+
+# The address the socket server listens on. It will get the value returned from
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+# FORMAT:
+# listeners = listener_name://host_name:port
+# EXAMPLE:
+# listeners = PLAINTEXT://your.host.name:9092
+#listeners=PLAINTEXT://:9092,CONTROLLER://:9093
+#listeners=PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092,CONTROLLER://localhost:9093
+listeners=PLAINTEXT://:29092,PLAINTEXT_HOST://:9092,CONTROLLER://:9093
+
+inter.broker.listener.name=PLAINTEXT
+#KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
+
+# Hostname and port the broker will advertise to producers and consumers. If not set,
+# it uses the value for "listeners" if configured. Otherwise, it will use the value
+# returned from java.net.InetAddress.getCanonicalHostName().
+#advertised.listeners=PLAINTEXT://localhost:9092
+advertised.listeners=PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+
+# Listener, host name, and port for the controller to advertise to the brokers. If
+# this server is a controller, this listener must be configured.
+controller.listener.names=CONTROLLER
+
+# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
+#listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+listener.security.protocol.map=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+# ,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+
+# The number of threads that the server uses for receiving requests from the network and sending responses to the network
+num.network.threads=3
+
+# The number of threads that the server uses for processing requests, which may include disk I/O
+num.io.threads=8
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes=102400
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes=102400
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes=104857600
+
+
+############################# Log Basics #############################
+
+# A comma separated list of directories under which to store log files
+log.dirs=/tmp/kraft-combined-logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions=3
+
+# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+# This value is recommended to be increased for installations with data dirs located in RAID array.
+num.recovery.threads.per.data.dir=1
+
+############################# Internal Topic Settings #############################
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.
+offsets.topic.replication.factor=1
+transaction.state.log.replication.factor=1
+transaction.state.log.min.isr=1
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+# 1. Durability: Unflushed data may be lost if you are not using replication.
+# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+#log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+#log.flush.interval.ms=1000
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion due to age
+log.retention.hours=168
+
+# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
+# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
+#log.retention.bytes=1073741824
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+log.segment.bytes=1073741824
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms=300000
+
+### Compression ###
+#compression.type=snappy
+#linger.ms=5
+
+
+### record size ###
+message.max.bytes=10485760
+replica.fetch.max.bytes=10485760
+
+#log.cleanup.policy=compact
+#auto.create.topics.enable=false
diff --git a/docs/kafkatorio-platform/docker-compose.yml b/docs/kafkatorio-platform/docker-compose.yml
new file mode 100644
index 0000000..5826700
--- /dev/null
+++ b/docs/kafkatorio-platform/docker-compose.yml
@@ -0,0 +1,140 @@
+version: "3.9"
+
+services:
+
+ ## Factorio game server that players will connect to
+ factorio-server:
+ image: factoriotools/factorio:${FACTORIO_VERSION}
+ container_name: "factorio-server"
+ restart: unless-stopped
+ profiles: [ "factorio", "kafkatorio" ]
+ networks:
+ - kafkatorio-infra-nw
+ ports:
+ - "34198:34197/udp" # factorio
+ - "27016:27015/tcp" # rcon
+ volumes:
+ - "./factorio-server:/factorio"
+ logging:
+ # set up syslog, so logs can be forwarded to Kafkatorio
+ driver: syslog
+ options:
+ syslog-address: ${SYSLOG_ADDRESS}
+ syslog-format: "rfc5424"
+ # set the tag to be a JWT, containing the server short-ID
+ tag: ${KAFKATORIO_TOKEN}
+
+
+ ## Kafkatorio Web Map host
+ web-map:
+ image: ${REGISTRY_HOST}/dev.adamko.kafkatorio/web-map:${KAFKATORIO_VERSION}
+ container_name: web-map
+ profiles: [ "infra", "kafkatorio" ]
+ restart: unless-stopped
+ networks:
+ - kafkatorio-infra-nw
+
+
+ ## Process Kafkatorio events
+ events-processors:
+ image: ${REGISTRY_HOST}/dev.adamko.kafkatorio/events-processors:${KAFKATORIO_VERSION}
+ container_name: events-processors
+ restart: unless-stopped
+ profiles: [ "infra", "kafkatorio" ]
+ environment:
+ KAFKA_BOOTSTRAP_SERVERS: "kafka:29092"
+ networks:
+ - kafkatorio-infra-nw
+ depends_on:
+ - infra-kafka
+ secrets:
+ - source: kafkatorio_config
+ target: /.secret.config.yml
+
+
+ ## Receives packets from the Factorio Server, forwarded via Docker syslog-plugin
+ events-server-syslog:
+ image: ${REGISTRY_HOST}/dev.adamko.kafkatorio/events-server-syslog:${KAFKATORIO_VERSION}
+ container_name: events-server-syslog
+ restart: unless-stopped
+ profiles: [ "infra", "kafkatorio" ]
+ environment:
+ KAFKA_BOOTSTRAP_SERVERS: "kafka:29092"
+ SOCKET_SERVER_HOST: "0.0.0.0"
+ SOCKET_SERVER_PORT: "9002"
+ ports:
+ - "12035:9002" # syslog socket
+ networks:
+ - kafkatorio-infra-nw
+ depends_on:
+ - infra-kafka
+ - events-processors
+ secrets:
+ - source: kafkatorio_config
+ target: /.secret.config.yml
+
+
+ ## Hosts map-tiles and forwards websocket messages to the web-map browsers
+ events-server-web:
+ image: ${REGISTRY_HOST}/dev.adamko.kafkatorio/events-server-web:${KAFKATORIO_VERSION}
+ container_name: events-server-web
+ restart: unless-stopped
+ profiles: [ "infra", "kafkatorio" ]
+ environment:
+ KAFKA_BOOTSTRAP_SERVERS: "kafka:29092"
+ WEB_SERVER_PORT: "80"
+ ports:
+ - "12080:80" # http and ws
+ networks:
+ - kafkatorio-infra-nw
+ depends_on:
+ - infra-kafka
+ - events-processors
+ secrets:
+ - source: kafkatorio_config
+ target: /.secret.config.yml
+
+
+ ## An instance of Kafka
+ infra-kafka:
+ image: ${REGISTRY_HOST}/dev.adamko.kafkatorio/kafka-kraft:${KAFKATORIO_VERSION}
+ container_name: infra-kafka
+ restart: unless-stopped
+ profiles: [ "infra", "kafka" ]
+ networks:
+ kafkatorio-infra-nw:
+ aliases: [ kafka ]
+ volumes:
+ - "./config/kafka-server.properties:/kafka/server.properties"
+ ports:
+ # note: other Docker containers should use 29092
+ - "9092:9092"
+ - "9093:9093"
+
+
+ ## web-based GUI for Kafka
+ infra-kafka-ui:
+ image: provectuslabs/kafka-ui:v0.4.0
+ container_name: infra-kafka-ui
+ restart: unless-stopped
+ profiles: [ "infra", "kafka" ]
+ networks:
+ - kafkatorio-infra-nw
+ ports:
+ - "18090:8080"
+ depends_on:
+ - infra-kafka
+ environment:
+ KAFKA_CLUSTERS_0_NAME: "${COMPOSE_PROJECT_NAME}"
+ KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka:29092"
+
+
+
+secrets:
+ kafkatorio_config:
+ file: ./.secret.kafkatorio-config.yml
+
+
+networks:
+ kafkatorio-infra-nw:
+ kafkatorio-game-nw:
diff --git a/docs/kafkatorio-platform/example.env b/docs/kafkatorio-platform/example.env
new file mode 100644
index 0000000..4339796
--- /dev/null
+++ b/docs/kafkatorio-platform/example.env
@@ -0,0 +1,10 @@
+# Docker Compose containers config
+
+COMPOSE_PROJECT_NAME=Kafkatorio
+
+KAFKATORIO_TOKEN=
+KAFKATORIO_VERSION=0.9.15
+
+REGISTRY_HOST=localhost
+
+SYSLOG_ADDRESS=tcp://www.my-website.com:12345
diff --git a/docs/kafkatorio-platform/example.secret.kafkatorio-config.yml b/docs/kafkatorio-platform/example.secret.kafkatorio-config.yml
new file mode 100644
index 0000000..4e5237b
--- /dev/null
+++ b/docs/kafkatorio-platform/example.secret.kafkatorio-config.yml
@@ -0,0 +1,10 @@
+# Kafkatorio server config
+
+jwt-secret: "correct horse battery staple"
+
+# Define the servers that have the Kafkatorio Mod installed.
+kafkatorio-servers:
+ # The key is the short ID, used to uniquely identify a server (used in URLs).
+ # The value is a description.
+ short-id-123: "Name of server"
+ another-id-456: "Name of another server"
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index dab8d4c..9fb61b9 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -47,7 +47,7 @@ marykRocksDb = "7.3.1-2" # https://search.maven.org/artifact/io.m
klogging = "0.4.9"
-nimbusJoseJwt = "9.25.6" # https://search.maven.org/artifact/com.nimbusds/nimbus-jose-jwt
+nimbusJoseJwt = "9.25.6" # https://search.maven.org/artifact/com.nimbusds/nimbus-jose-jwt
## npm ##