From 2ad0c2848c437f26b649f17e40c94d6b88324b0e Mon Sep 17 00:00:00 2001 From: Adam <897017+aSemy@users.noreply.github.com> Date: Mon, 27 Dec 2021 12:24:55 +0100 Subject: [PATCH] giving up on kafka connect websockets, it doesn't work and there's no error messages and the documentaiton sucks --- build.gradle.kts | 2 +- modules/infra-kafka-connect/build.gradle.kts | 3 +- .../infra-kafka-connect/src/main/docker/.env | 2 +- .../src/main/docker/docker-compose.yml | 3 +- .../kafkaconnect/kafkaConnectAdmin.kt | 4 +- .../kafkaconnect/websocketClient.kt | 4 +- .../configs/ckc-websocket-lua-objects.json5 | 130 ++++++++++++++++++ ...son5 => websocket-async-lua-objects.json5} | 22 ++- settings.gradle.kts | 2 +- 9 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 rename modules/infra-kafka-connect/src/main/resources/configs/{websocket-sink-lua-topics.json5 => websocket-async-lua-objects.json5} (85%) diff --git a/build.gradle.kts b/build.gradle.kts index a28bcdc..94fee6c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -38,7 +38,7 @@ val startInfra by tasks.registering { dependsOn( ":modules:infra-kafka-pipe:processRun", ":modules:infra-kafka-cluster:processRun", - ":modules:infra-kafka-connect:processRun", +// ":modules:infra-kafka-connect:processRun", ) } diff --git a/modules/infra-kafka-connect/build.gradle.kts b/modules/infra-kafka-connect/build.gradle.kts index 871f9c1..a233462 100644 --- a/modules/infra-kafka-connect/build.gradle.kts +++ b/modules/infra-kafka-connect/build.gradle.kts @@ -27,6 +27,7 @@ dependencies { kotlin("reflect") kafkaConnector("camel-ahc-ws-kafka-connector") + kafkaConnector("camel-websocket-kafka-connector") implementation("com.github.adamko-dev:json5-kotlin:2.0.3") @@ -49,7 +50,7 @@ val dockerBuildKafkaConnect by tasks.registering(Exec::class) { workingDir = dockerSrcDir.asFile commandLine = parseSpaceSeparatedArgs(""" docker-compose build connect """) } -tasks.assemble { dependsOn(dockerBuildKafkaConnect) } +//tasks.assemble { dependsOn(dockerBuildKafkaConnect) } val downloadCamelConnectors by tasks.registering(Sync::class) { diff --git a/modules/infra-kafka-connect/src/main/docker/.env b/modules/infra-kafka-connect/src/main/docker/.env index a1fb7e3..3af8d5f 100644 --- a/modules/infra-kafka-connect/src/main/docker/.env +++ b/modules/infra-kafka-connect/src/main/docker/.env @@ -1,4 +1,4 @@ # Do not edit manually. This file was created with task 'dockerEnv' COMPOSE_PROJECT_NAME=kafkatorio CONNECTOR_ID=kafkatorio-connect -KAFKA_CONNECT_VERSION=7.0.0 +KAFKA_CONNECT_VERSION=7.0.1 diff --git a/modules/infra-kafka-connect/src/main/docker/docker-compose.yml b/modules/infra-kafka-connect/src/main/docker/docker-compose.yml index f152ae6..278ba19 100644 --- a/modules/infra-kafka-connect/src/main/docker/docker-compose.yml +++ b/modules/infra-kafka-connect/src/main/docker/docker-compose.yml @@ -16,7 +16,8 @@ services: # CAMEL_CONNECTOR: ${CAMEL_FILE_CONNECTOR} ports: - "8083:8083" - - "8095:8095" + - "8095:8095" # ws-async (client?) + - "9292:9292" # websocket server(?) # volumes: # - "./connect/input:/warehouse/input" # - "./connect/components:/usr/share/confluent-hub-components" diff --git a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt b/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt index 6a68965..e8eccfb 100644 --- a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt +++ b/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/kafkaConnectAdmin.kt @@ -24,7 +24,7 @@ fun main() { val connectorHostUrl = Uri.of("http://localhost:8083") - val configFile = Resources.loadResource("/configs/websocket-sink-lua-topics.json5") + val configFile = Resources.loadResource("/configs/ckc-websocket-lua-objects.json5") val jsonObject: JsonObject = j5.decodeObject(configFile.readText()) println("json5 connector config: $jsonObject") @@ -59,7 +59,7 @@ fun main() { //val updateConnectors by tasks.registering { // group = project.name // -// val configFile = layout.projectDirectory.file("configs/websocket-sink-lua-topics.json5") +// val configFile = layout.projectDirectory.file("configs/websocket-async-lua-objects.json5") // inputs.file(configFile) // // doLast { diff --git a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt b/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt index a5f6609..b7ee898 100644 --- a/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt +++ b/modules/infra-kafka-connect/src/main/kotlin/dev/adamko/kafkatorio/kafkaconnect/websocketClient.kt @@ -22,11 +22,11 @@ fun main() { val nonBlockingClient = WebsocketClient.nonBlocking( - uri = Uri.of("ws://localhost:8095/ws"), + uri = Uri.of("ws://localhost:9292/lua-objects"), onError = { it.printStackTrace() }, onConnect = { ws -> println("onConnect, ws.upgradeRequest ${ws.upgradeRequest}") - ws.send(WsMessage("client sent on connection")) +// ws.send(WsMessage("client sent on connection")) } ) diff --git a/modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 b/modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 new file mode 100644 index 0000000..c96e8c8 --- /dev/null +++ b/modules/infra-kafka-connect/src/main/resources/configs/ckc-websocket-lua-objects.json5 @@ -0,0 +1,130 @@ +{ + "name": "ckc-websocket-lua-objects", + "connector.class": "org.apache.camel.kafkaconnector.websocket.CamelWebsocketSinkConnector", + "topics.regex": "^Lua.*$", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.storage.StringConverter", + // The hostname. The default value is 0.0.0.0. Setting this option on the component will use the + // component configured value as default. + "camel.sink.path.host": "0.0.0.0", +// "camel.sink.path.host": "host.docker.internal", + // The port number. The default value is 9292. Setting this option on the component will use the + // component configured value as default. + "camel.sink.path.port": "9292", + // Name of the websocket channel to use + "camel.sink.path.resourceUri": "lua-objects", + "camel.sink.endpoint.sendToAll": "true", + "camel.component.websocket.host":"0.0.0.0", + "camel.component.websocket.port": "8084", + // Can be used to set the size in bytes that the websocket created by the websocketServlet may + // be accept before closing. (Default is -1 - or unlimited) + // "camel.sink.endpoint.maxBinaryMessageSize": -1, + + // Whether the producer should be started lazy (on the first message). By starting lazy you can + // use this to allow CamelContext and routes to startup in situations where a producer may otherwise + // fail during starting and cause the route to fail being started. By deferring this startup to be + // lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. + // Beware that when the first message is processed then creating and starting the producer may take a little + // time and prolong the total processing time of the processing. + // "camel.sink.endpoint.lazyStartProducer": false, + + // Timeout in millis when sending to a websocket channel. The default timeout is 30000 (30 seconds). + // "camel.sink.endpoint.sendTimeout": 30000, + + // To send to all websocket subscribers. Can be used to configure on endpoint level, instead of + // having to use the WebsocketConstants.SEND_TO_ALL header on the message. + // "camel.sink.endpoint.sendToAll": , + + // Set the buffer size of the websocketServlet, which is also the max frame byte size (default + // 8192) + // "camel.sink.endpoint.bufferSize": 8192, + + // Set the time in ms that the websocket created by the websocketServlet may be idle before + // closing. (default is 300000) + // "camel.sink.endpoint.maxIdleTime": 300000, + + // Can be used to set the size in characters that the websocket created by the websocketServlet + // may be accept before closing. + // "camel.sink.endpoint.maxTextMessageSize": , + + // Can be used to set the minimum protocol version accepted for the websocketServlet. + // (Default 13 - the RFC6455 version) + // "camel.sink.endpoint.minVersion": 13, + + // The CORS allowed origins. Use to allow all. + // "camel.sink.endpoint.allowedOrigins": , + + // Whether to enable CORS + // "camel.sink.endpoint.crossOriginFilterOn": false, + + // Context path for filtering CORS + // "camel.sink.endpoint.filterPath": , + + // If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX + // support for more details. + // "camel.sink.endpoint.enableJmx": false, + + // To configure security using SSLContextParameters + // "camel.sink.endpoint.sslContextParameters": , + + // The hostname. The default value is 0.0.0.0 + // "camel.component.websocket.host": "0.0.0.0", + + // The port number. The default value is 9292 + // "camel.component.websocket.port": 9292, + + // Whether the producer should be started lazy (on the first message). By starting lazy you can + // use this to allow CamelContext and routes to startup in situations where a producer may otherwise + // fail during starting and cause the route to fail being started. By deferring this startup to be + // lazy then the startup failure can be handled during routing messages via Camel's routing error handlers. + // Beware that when the first message is processed then creating and starting the producer may take a little + // time and prolong the total processing time of the processing. + // "camel.component.websocket.lazyStartProducer": false, + + // Whether autowiring is enabled. This is used for automatic autowiring options (the option must + // be marked as autowired) by looking up in the registry to find if there is a single instance + // of matching type, which then gets configured on the component. This can be used for automatic + // configuring JDBC data sources, JMS connection factories, AWS Clients, etc. + // "camel.component.websocket.autowiredEnabled": true, + + // If this option is true, Jetty JMX support will be enabled for this endpoint. See Jetty JMX + // support for more details. + // "camel.component.websocket.enableJmx": false, + + // To set a value for maximum number of threads in server thread pool. MaxThreads\/minThreads + // or threadPool fields are required due to switch to Jetty9. The default values for maxThreads + // is 1 2 noCores. + // "camel.component.websocket.maxThreads": , + + // To set a value for minimum number of threads in server thread pool. MaxThreads\/minThreads + // or threadPool fields are required due to switch to Jetty9. The default values for minThreads + // is 1. + // "camel.component.websocket.minThreads": , + + // This is a comma-separated list of subprotocols that are supported by the application. The list + // is in priority order. The first subprotocol on this list that is proposed by the client is + // the one that will be accepted. If no subprotocol on this list is proposed by the client, then the + // websocket connection is refused. The special value 'any' means that any subprotocol is acceptable. 'any' + // can be used on its own, or as a failsafe at the end of a list of more specific protocols. 'any' will also + // match the case where no subprotocol is proposed by the client. + // "camel.component.websocket.subprotocol": "any", + + // To use a custom thread pool for the server. MaxThreads\/minThreads or threadPool fields are + // required due to switch to Jetty9. + // "camel.component.websocket.threadPool": , + + // To configure security using SSLContextParameters + // "camel.component.websocket.sslContextParameters": , + + // The password for the keystore when using SSL. + // "camel.component.websocket.sslKeyPassword": , + + // The path to the keystore. + // "camel.component.websocket.sslKeystore": , + + // The password when using SSL. + // "camel.component.websocket.sslPassword": , + + // Enable usage of global SSL context parameters. + // "camel.component.websocket.useGlobalSslContextParameters": false, +} diff --git a/modules/infra-kafka-connect/src/main/resources/configs/websocket-sink-lua-topics.json5 b/modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5 similarity index 85% rename from modules/infra-kafka-connect/src/main/resources/configs/websocket-sink-lua-topics.json5 rename to modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5 index d11fdcc..19837c3 100644 --- a/modules/infra-kafka-connect/src/main/resources/configs/websocket-sink-lua-topics.json5 +++ b/modules/infra-kafka-connect/src/main/resources/configs/websocket-async-lua-objects.json5 @@ -1,6 +1,6 @@ //Exchange data with external Websocket servers using Async Http Client { - name: "CKC-async-ws-lua-objects", + name: "websocket-async-lua-objects", "connector.class": "org.apache.camel.kafkaconnector.ahcws.CamelAhcwsSinkConnector", "tasks.max": 1, //use the kafka converters that better suit your needs, these are just defaults: @@ -14,17 +14,29 @@ // mandatory properties (for a complete properties list see the connector documentation): // The URI to use such as http://hostname:port/path - "camel.sink.path.httpUri": "localhost:8095/ws", - // If the option is true, then the Exchange.HTTP_URI header is ignored, and use the endpoint's URI for request. You may also set the throwExceptionOnFailure to be false to let the AhcProducer send all the fault response back. + "camel.sink.path.httpUri": "host.docker.internal:8095/ws/lua", + // If the option is true, then the Exchange.HTTP_URI header is ignored, and use the endpoint's + // URI for request. You may also set the throwExceptionOnFailure to be false to let the + // AhcProducer send all the fault response back. // camel.sink.endpoint.bridgeEndpoint= + // The initial in-memory buffer size used when transferring data between Camel and AHC Client. // camel.sink.endpoint.bufferSize= // To use a custom HeaderFilterStrategy to filter header to and from Camel message. // camel.sink.endpoint.headerFilterStrategy= - // Option to disable throwing the AhcOperationFailedException in case of failed responses from the remote server. This allows you to get all responses regardless of the HTTP status code. + // Option to disable throwing the AhcOperationFailedException in case of failed responses from + // the remote server. This allows you to get all responses regardless of the HTTP status code. // camel.sink.endpoint.throwExceptionOnFailure= - // If enabled and an Exchange failed processing on the consumer side, and if the caused Exception was send back serialized in the response as a application\/x-java-serialized-object content type (for example using Jetty or Servlet Camel components). On the producer side the exception will be deserialized and thrown as is, instead of the AhcOperationFailedException. The caused exception is required to be serialized. This is by default turned off. If you enable this then be aware that Java will deserialize the incoming data from the request to Java and that can be a potential security risk. + // If enabled and an Exchange failed processing on the consumer side, and if the caused + + // Exception was send back serialized in the response as a application\/x-java-serialized-object + // content type (for example using Jetty or Servlet Camel components). On the producer side the + // exception will be deserialized and thrown as is, instead of the AhcOperationFailedException. + // The caused exception is required to be serialized. This is by default turned off. + // If you enable this then be aware that Java will deserialize the incoming data from the + // request to Java and that can be a potential security risk. // camel.sink.endpoint.transferException= + // Define if the Connection Close header has to be added to HTTP Request. This parameter is false by default // camel.sink.endpoint.connectionClose= // Configure a cookie handler to maintain a HTTP session diff --git a/settings.gradle.kts b/settings.gradle.kts index 3af2206..a99134d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,7 +11,7 @@ include( ":modules:infra-kafka-cluster", ":modules:infra-kafka-pipe", - ":modules:infra-kafka-connect", +// ":modules:infra-kafka-connect", ":modules:kt-rcon", ":modules:web-map",