Documentation updates

Modbus adapter configuration check
File Adapter template support
OPCUA target monitor output fix

Author: arieleeuw

Diff for: README.md

@@ -91,7 +91,7 @@ At first, we have to download and extract the SFC bundles. These are precompiled
 
 ```shell
 # Define sfc version and directory
-export VERSION=$(git describe --tags --abbrev=0)
+export VERSION=$(curl -s "https://api.github.com/repos/aws-samples/shopfloor-connectivity/tags" | jq -r '.[0].name')
 export SFC_DEPLOYMENT_DIR="./sfc"
 ```
 

Diff for: RELEASE NOTES.md

@@ -1,4 +1,13 @@
 # Release Notes:
+## Version 1.8.5, 4 march February 2025
+
+- Documentation updates
+- OPCUA Target monitor output fix
+- Support for [templates](./docs/core/target-configuration.md#template) in File Target adapter 
+- Modbus TCP configuration validation update
+- OPCUA Target handling of unsigned datatypes
+- Fix in quickstart documentation
+
 ## Version 1.8.4, 26 February 2025
 
 - New [OPCUA Writer](./docs/targets/opcua-writer.md) Target

Diff for: adapters/modbus/src/main/kotlin/com/amazonaws/sfc/modbus/adapter/ModbusDevice.kt

@@ -63,15 +63,8 @@ class ModbusDevice(
 
         val reader = launch("Reader") {
 
-            val logTrace = logger.get()?.getCtxTraceLog(ModbusDevice::class.java.simpleName, "reader")
-
             try {
 
-                // get exclusive access to the device
-                logTrace?.invoke("Acquiring lock on device \"${configuration.sourceAdapterDevice}\" on adapter \"${configuration.protocolAdapterID}\" to read data for source \"$sourceID\"")
-                modbus.modbusDevice.lock()
-                logTrace?.invoke("Acquired lock on device \"${configuration.sourceAdapterDevice}\"  on adapter \"${configuration.protocolAdapterID}\" for source \"${configuration.name}\"")
-
                 // number of requests that can be sent to a without a response received for it, guarded by this semaphore
                 val requestSlots = Semaphore(permits = adapter.get()?.requestDepth(configuration)?.toInt() ?: 1)
 
@@ -127,9 +120,6 @@ class ModbusDevice(
             } catch (e: Exception) {
                 readResult = SourceReadError(e.toString())
 
-            } finally {
-                modbus.modbusDevice.unlock()
-                logTrace?.invoke("Released lock on device\"${configuration.sourceAdapterDevice}\" on adapter \"${configuration.protocolAdapterID}\"")
             }
         }
 

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/awsiot/AwsIotCredentialProviderClientConfiguration.kt

@@ -12,7 +12,6 @@ import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_PRIVATE_KEY
 import com.amazonaws.sfc.data.JsonHelper
 import com.amazonaws.sfc.log.Logger.Companion.HIDDEN_VALUE
 import com.google.gson.annotations.SerializedName
-import kotlin.Throws
 
 @ConfigurationClass
 class AwsIotCredentialProviderClientConfiguration : ClientConfiguration(), Validate {

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/BaseSourceConfiguration.kt

@@ -9,7 +9,6 @@ package com.amazonaws.sfc.config
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_DESCRIPTION
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_NAME
 import com.google.gson.annotations.SerializedName
-import kotlin.Throws
 
 /**
  * Base class with minimum set of attributes for an SFC input sourceService

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/ChannelConfiguration.kt

@@ -12,7 +12,6 @@ import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_NAME
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_SPREAD_CHANNEL_VALUE
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_VALUE_FILTER
 import com.google.gson.annotations.SerializedName
-import kotlin.Throws
 
 /**
  * Base class with minimum set of attributes for a channel of a source.

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/ClientConfiguration.kt

@@ -11,7 +11,6 @@ import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_CERTIFICATE
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_PRIVATE_KEY
 import com.google.gson.annotations.SerializedName
 import java.io.File
-import kotlin.Throws
 
 @ConfigurationClass
 open class ClientConfiguration : Validate {

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/ConfigurationException.kt

@@ -7,7 +7,6 @@ package com.amazonaws.sfc.config
 
 import com.amazonaws.sfc.data.JsonHelper
 import com.amazonaws.sfc.data.JsonHelper.Companion.gsonExtended
-import kotlin.Throws
 
 interface Validate {
     @Throws(ConfigurationException::class)

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/HealthProbeConfiguration.kt

@@ -7,7 +7,6 @@ package com.amazonaws.sfc.config
 
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_ACTIVE
 import com.google.gson.annotations.SerializedName
-import kotlin.Throws
 import kotlin.time.Duration
 import kotlin.time.DurationUnit
 import kotlin.time.toDuration

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/ScheduleConfiguration.kt

@@ -13,7 +13,6 @@ import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_META_DATA
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_NAME
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_TARGETS
 import com.google.gson.annotations.SerializedName
-import kotlin.Throws
 import kotlin.time.Duration
 import kotlin.time.DurationUnit
 import kotlin.time.toDuration

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/ServerConfiguration.kt

@@ -10,7 +10,6 @@ import com.amazonaws.sfc.crypto.CertificateConfiguration.Companion.CONFIG_CERT_E
 import com.amazonaws.sfc.service.ServerConnectionType
 import com.google.gson.annotations.SerializedName
 import java.io.File
-import kotlin.Throws
 import kotlin.time.Duration
 import kotlin.time.DurationUnit
 import kotlin.time.toDuration

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/ServiceConfiguration.kt

@@ -10,7 +10,6 @@ import com.amazonaws.sfc.config.ServerConfiguration.Companion.CONFIG_SERVER_ADDR
 import com.amazonaws.sfc.log.LogLevel
 import com.amazonaws.sfc.metrics.MetricsConfiguration
 import com.google.gson.annotations.SerializedName
-import kotlin.Throws
 import kotlin.time.Duration
 import kotlin.time.DurationUnit
 import kotlin.time.toDuration

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/config/SourceConfiguration.kt

@@ -12,7 +12,6 @@ import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_META_DATA
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_NAME
 import com.amazonaws.sfc.config.BaseConfiguration.Companion.CONFIG_SPREAD_CHANNEL_VALUE
 import com.google.gson.annotations.SerializedName
-import kotlin.Throws
 
 /**
  * Source configuration with attributes for a source as used by the SFC core controller

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/crypto/KeyHelpers.kt

@@ -19,7 +19,6 @@ import java.security.interfaces.ECPrivateKey
 import java.security.interfaces.RSAPrivateCrtKey
 import java.security.spec.*
 import java.util.*
-import kotlin.Throws
 
 // -----BEGIN RSA PRIVATE KEY-----
 @Suppress("MemberVisibilityCanBePrivate") object KeyHelpers {

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/data/OutputTransformation.kt

@@ -14,11 +14,7 @@ import org.apache.velocity.app.Velocity
 import org.apache.velocity.app.VelocityEngine
 import org.apache.velocity.runtime.resource.loader.StringResourceLoader
 import org.apache.velocity.runtime.resource.util.StringResourceRepository
-import org.apache.velocity.tools.generic.CollectionTool
-import org.apache.velocity.tools.generic.ContextTool
-import org.apache.velocity.tools.generic.DateTool
-import org.apache.velocity.tools.generic.MathTool
-import org.apache.velocity.tools.generic.NumberTool
+import org.apache.velocity.tools.generic.*
 import java.io.File
 import java.io.StringWriter
 import java.nio.file.Files

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/data/SourceDataMultiValuesStore.kt

@@ -4,7 +4,7 @@
 
 package com.amazonaws.sfc.data
 
-import io.ktor.util.collections.ConcurrentSet
+import io.ktor.util.collections.*
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/data/SourcesValuesAsFlow.kt

@@ -134,6 +134,7 @@ class SourcesValuesAsFlow(
                 if (!initialRead && duration > interval) {
                     log.warning("Read cycle took ${duration.inWholeMilliseconds.toDuration(DurationUnit.MILLISECONDS)}, " +
                             "which is more than read interval of $interval ${if (schedule != null)" for schedule \"$schedule\"" else ""}")
+
                 } else {
                     log.trace("Read cycle took $duration")
                     runBlocking {

Diff for: core/sfc-core/src/main/kotlin/com/amazonaws/sfc/util/UrlReaderCache.kt

@@ -15,11 +15,9 @@ import kotlinx.coroutines.runBlocking
 import java.io.File
 import java.io.File.separator
 import java.nio.charset.Charset
-import java.nio.file.FileSystem
 import java.security.MessageDigest
 import java.time.Clock
 import java.time.Instant
-import kotlin.collections.toString
 import kotlin.time.Duration
 
 

Diff for: core/sfc-ipc/src/main/kotlin/com/amazonaws/sfc/ipc/IpcAdapterService.kt

@@ -84,10 +84,15 @@ class IpcAdapterService(
 
         val log = logger.getCtxLoggers(className, "start")
 
-        grpcServer.start()
-        val addressAndPort = grpcServer.listenSockets.first() as InetSocketAddress
-        val addressAndPortStr = "${addressAndPort.address.hostAddress}:${addressAndPort.port}"
-        log.info("IPC protocol service started, listening on $addressAndPortStr, connection type is ${serverConfig.serverConnectionType}")
+        try {
+            grpcServer.start()
+            val addressAndPort = grpcServer.listenSockets.first() as InetSocketAddress
+            val addressAndPortStr = "${addressAndPort.address.hostAddress}:${addressAndPort.port}"
+            log.info("IPC protocol service started, listening on $addressAndPortStr, connection type is ${serverConfig.serverConnectionType}")
+        }catch (e: Exception) {
+            log.error("Failed to start server, $e")
+            exitProcess(1)
+        }
 
         Runtime.getRuntime().addShutdownHook(
             Thread {
@@ -321,7 +326,7 @@ class IpcAdapterService(
                     service.restartIfInactive()
                 }
                 service
-            } catch (e: Exception) {
+            } catch ( _ : Exception) {
                 null
             }
         healthProbeService?.start()

Diff for: core/sfc-ipc/src/main/kotlin/com/amazonaws/sfc/ipc/IpcSourceReader.kt

@@ -298,7 +298,7 @@ class IpcSourceReader(
     private fun adaptersToServerMap(configRaw: Map<*, *>) = allAdaptersForType(configRaw).map {
         val adapter = it.value as Map<*, *>
         val adapterID = it.key as String
-        val serverID = adapter[CONFIG_PROTOCOL_ADAPTER_SERVER] as String
+        val serverID = adapter[CONFIG_PROTOCOL_ADAPTER_SERVER] as String?
         adapterID to serverID
     }.toMap()
 

Diff for: core/sfc-main/src/main/kotlin/com/amazonaws/sfc/MainControllerService.kt

@@ -118,7 +118,7 @@ class MainControllerService(
                     stopScheduleControllers(),
                 ).joinAll()
             }
-        } catch (e: Exception) {
+        } catch (_: Exception) {
             logger.getCtxWarningLog(className, "stop")("Timeout stopping service")
         }
 
@@ -348,15 +348,22 @@ class MainControllerService(
     }
 
     private fun buildSourceValuesReader(schedule: ScheduleConfiguration, sourceID: String, adapterID: String): SourceValuesReader? {
-
+        val log = logger.getCtxLoggers(className, "buildSourceValuesReader")
         val protocolConfiguration = configuration.protocolAdapters[adapterID]
-        val protocolServerConfiguration = configuration.protocolAdapterServers[protocolConfiguration?.protocolAdapterServer]
+        val protocolAdapterServer = protocolConfiguration?.protocolAdapterServer
+        val protocolServerConfiguration = if (protocolAdapterServer!= null) {
+            if (!configuration.protocolAdapterServers.keys.contains(protocolAdapterServer)){
+                log.error("Protocol adapter server \"$protocolAdapterServer\" for protocol \"$adapterID\" does not exist, valid servers are ${configuration.protocolAdapterServers.keys}")
+                null
+            }else{
+                configuration.protocolAdapterServers[protocolAdapterServer]
+            }
+        } else null
 
         return if (protocolServerConfiguration != null) {
             createIpcReader(configReader, configuration, adapterID, schedule, logger) as SourceValuesReader
         } else {
             if (protocolConfiguration?.protocolAdapterType !in configuration.protocolAdapterTypes) {
-                val log = logger.getCtxLoggers(className, "buildSourceValuesReader")
                 log.error(
                     "Protocol type ${protocolConfiguration?.protocolAdapterType} for protocol \"$adapterID\" used in source \"Source \"$sourceID\" does not exist, " +
                             "valid types are ${configuration.protocolAdapterTypes.keys}"

Diff for: deployment/greengrass-sfc-components/local-build-as-components-py/buildSfcComponentRecipes.py

@@ -1,4 +1,8 @@
-import os, stat, argparse, shutil, json
+import argparse
+import json
+import os
+import shutil
+import stat
 
 # cli arg definitions...
 parser = argparse.ArgumentParser()

Diff for: docs/README.md

@@ -4,7 +4,7 @@
 
 The documentation provides a comprehensive overview of SFC's architecture, capabilities, and deployment options for industrial data collection and ingestion to AWS.
 
-## **Summary **
+## **Summary**
 
 Shop Floor Connectivity (SFC) is a data ingestion technology for collecting industrial data and delivering it to AWS services.
 

Diff for: docs/adapters/ads.md

@@ -13,13 +13,31 @@ Key configuration points:
 
 The adapter supports reading from various Beckhoff PLC data types and handles the protocol-specific details of ADS communication, making it straightforward to integrate Beckhoff data into your existing SFC data collection infrastructure.
 
+In order to use this adapter as in [in-process](../sfc-running-adapters.md#running-protocol-adapters-in-process) type adapter the type must be added to the [AdapterTypes](../core/sfc-configuration.md#AdapterTypes) section in the [SFC configuration file](../core/sfc-configuration.md).
+
+```json
+"AdapterTypes" :{
+  "ADS" : {
+    "JarFiles" : ["<location of deployment>/ads/lib"]
+  },
+  "FactoryClassName" : "com.amazonaws.sfc.ads.AdsAdapter"
+}
+```
+
 
 
+**Configuration:**
+
 - [AdsSourceConfiguration](#adssourceconfiguration)
+
 - [AdsChannelConfiguration](#adschannelconfiguration)
+
 - [AdsAdapterConfiguration](#adsadapterconfiguration)
+
 - [AdsDeviceConfiguration](#adsdeviceconfiguration)
 
+  
+
 ---
 
 ## AdsSourceConfiguration

Diff for: docs/adapters/j1939.md

@@ -9,16 +9,25 @@ An open source J1939 DBC file is available at [j1939](../../examples/j1939dbc/RE
 
 > **Note: This adapter requires SocketCAN and can only be used on POSIX-compliant operating systems like Linux.**
 
+In order to use this adapter as in [in-process](../sfc-running-adapters.md#running-protocol-adapters-in-process) type adapter the type must be added to the [AdapterTypes](../core/sfc-configuration.md#AdapterTypes) section in the [SFC configuration file](../core/sfc-configuration.md).
+
+```json
+"AdapterTypes" :{
+  "J1939" : {
+    "JarFiles" : ["<location of deployment>/ads/lib"]
+  },
+  "FactoryClassName" : "com.amazonaws.sfc.j1939.J1939Adapter"
+}
+```
+
+**Configuration:**
 
----
 - [J1939SourceConfiguration](#J1939sourceconfiguration)
 
 - [J1939ChannelConfiguration](#J1939channelconfiguration)
 
 - [J1939AdapterConfiguration](#J1939adapterconfiguration)
 
-  
-
 ---
 
 ## J1939SourceConfiguration

Diff for: docs/adapters/modbus.md

@@ -2,14 +2,27 @@
 
 The Modbus TCP protocol adapter enables communication with devices supporting the Modbus TCP protocol over TCP/IP networks. It supports standard Modbus functions for reading and writing coils, discrete inputs, holding registers, and input registers. Configure device connections using IP address and port, specify unit IDs, and define register addresses and data types for your channels. The adapter handles all protocol-specific details, making it easy to integrate Modbus device data into your SFC infrastructure.
 
+In order to use this adapter as an [in-process](file:///Applications/Typora.app/Contents/Resources/sfc-running-adapters.md#running-protocol-adapters-in-process) type adapter, the type must be added to the [AdapterTypes](file:///Applications/Typora.app/Contents/Resources/core/sfc-configuration.md#AdapterTypes) section in the [SFC configuration file](file:///Applications/Typora.app/Contents/Resources/core/sfc-configuration.md).
+
+```json
+"AdapterTypes" :{
+  "MODBUS-TCP" : {
+    "JarFiles" : ["<location of deployment>/modbus-tcp/lib"]
+  },
+  "FactoryClassName" : "com.amazonaws.sfc.modbus.tcp.ModbusTcpAdapter"
+}
+```
+
+
+
+**Configuration**:
+
 - [ModbusSourceConfiguration](#modbussourceconfiguration)
 - [ModbusOptimization](#modbusoptimization)
 - [ModbusChannelConfiguration](#modbuschannelconfiguration)
 - [ModbusTcpAdapterConfiguration](#modbustcpadapterconfiguration)
 - [ModbusTcpDeviceConfiguration](#modbustcpdeviceconfiguration)
 
----
-
 ## ModbusSourceConfiguration
 
 [SFC Configuration](../core/sfc-configuration.md) > [Sources](../core/sfc-configuration.md#sources) >  [Source](../core/source-configuration.md) 

Diff for: docs/adapters/mqtt.md

@@ -6,10 +6,19 @@ The MQTT source protocol adapter enables SFC to collect data from MQTT brokers b
 
 This adapter is particularly useful in IoT scenarios where devices and sensors publish their data to MQTT topics, allowing SFC to integrate seamlessly with existing MQTT-based infrastructure. It supports features like QoS levels, SSL/TLS security, and client authentication to ensure reliable and secure data collection from MQTT sources.
 
-MQTT Protocol adapter configuration.
+In order to use this adapter as in [in-process](../sfc-running-adapters.md#running-protocol-adapters-in-process) type adapter the type must be added to the [AdapterTypes](../core/sfc-configuration.md#AdapterTypes) section in the [SFC configuration file](../core/sfc-configuration.md).
 
+```json
+"AdapterTypes" :{
+  "MQTT" : {
+    "JarFiles" : ["<location of deployment>/mqtt/lib"]
+  },
+  "FactoryClassName" : "com.amazonaws.sfc.mqtt.MqttAdapter"
+}
+```
+
+**Configuration:**
 
----
 - [MqttSourceConfiguration](#mqttsourceconfiguration)
 - [MqttChannelConfiguration](#mqttchannelconfiguration)
 - [TopicNameMapping](#topicnamemapping)