diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 077d4da6c..76dc8a124 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -78,5 +78,9 @@ ojdbc11 23.7.0.25.01 + + io.projectreactor + reactor-core + \ No newline at end of file diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index c19477a7c..5020d135c 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -39,5 +39,6 @@ booking-agent-m365 semantickernel-spring-starter sk-presidio-sample + sk-database-sample diff --git a/samples/semantickernel-demos/sk-database-sample/.mvn/jvm.config b/samples/semantickernel-demos/sk-database-sample/.mvn/jvm.config new file mode 100644 index 000000000..32599cefe --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/.mvn/jvm.config @@ -0,0 +1,10 @@ +--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED diff --git a/samples/semantickernel-demos/sk-database-sample/.mvn/wrapper/maven-wrapper.properties b/samples/semantickernel-demos/sk-database-sample/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 000000000..6d3a56651 --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.3/apache-maven-3.9.3-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/samples/semantickernel-demos/sk-database-sample/README.md b/samples/semantickernel-demos/sk-database-sample/README.md new file mode 100644 index 000000000..79dabf83f --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/README.md @@ -0,0 +1,14 @@ +# Database Sample + +This sample demonstrates how to use Semantic Kernel with a database to perform various operations such as creating, +reading, updating, and deleting records. The sample uses MySQL as the database engine. + +# Prerequisites + +- Docker/Docker Compose + +# Running the Sample + +- Ensure you have compiled/installed the Semantic Kernel library. +- Copy env.example to .env and fill out the settings. +- Run `./buildAndRun.sh` to build and run the sample. diff --git a/samples/semantickernel-demos/sk-database-sample/buildAndRun.sh b/samples/semantickernel-demos/sk-database-sample/buildAndRun.sh new file mode 100755 index 000000000..fe3a1477d --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/buildAndRun.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -eux + +./mvnw package + +docker compose build && docker compose up \ No newline at end of file diff --git a/samples/semantickernel-demos/sk-database-sample/docker-compose.yml b/samples/semantickernel-demos/sk-database-sample/docker-compose.yml new file mode 100644 index 000000000..c4765255b --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/docker-compose.yml @@ -0,0 +1,27 @@ +version: '3.6' +services: + mysql: + hostname: mysql + image: mysql:8.4 + command: --mysql-native-password=ON + ports: + - "3306:3306" + environment: + - MYSQL_ROOT_PASSWORD= + - MYSQL_ALLOW_EMPTY_PASSWORD=true + - MYSQL_USER=a-user + - MYSQL_PASSWORD=a-password + - MYSQL_DATABASE=testdb + sk-database-sample: + image: "sk-database-sample" + depends_on: + - mysql + build: + context: . + dockerfile: ./sk-database-sample-dockerfile + secrets: + - ai-config +secrets: + ai-config: + file: ./.env + diff --git a/samples/semantickernel-demos/sk-database-sample/env.example b/samples/semantickernel-demos/sk-database-sample/env.example new file mode 100644 index 000000000..85a2b7439 --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/env.example @@ -0,0 +1,7 @@ +# If OpenAI +#CLIENT_KEY="" + +# If Azure OpenAI +#USE_AZURE_CLIENT=true +#CLIENT_ENDPOINT="" +#AZURE_CLIENT_KEY="" diff --git a/samples/semantickernel-demos/sk-database-sample/mvnw b/samples/semantickernel-demos/sk-database-sample/mvnw new file mode 100755 index 000000000..8d937f4c1 --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/mvnw @@ -0,0 +1,308 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.2.0 +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /usr/local/etc/mavenrc ] ; then + . /usr/local/etc/mavenrc + fi + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "$(uname)" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME + else + JAVA_HOME="/Library/Java/Home"; export JAVA_HOME + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --unix "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --unix "$CLASSPATH") +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && + JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="$(which javac)" + if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=$(which readlink) + if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then + if $darwin ; then + javaHome="$(dirname "\"$javaExecutable\"")" + javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" + else + javaExecutable="$(readlink -f "\"$javaExecutable\"")" + fi + javaHome="$(dirname "\"$javaExecutable\"")" + javaHome=$(expr "$javaHome" : '\(.*\)/bin') + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=$(cd "$wdir/.." || exit 1; pwd) + fi + # end of workaround + done + printf '%s' "$(cd "$basedir" || exit 1; pwd)" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + # Remove \r in case we run on Windows within Git Bash + # and check out the repository with auto CRLF management + # enabled. Otherwise, we may read lines that are delimited with + # \r\n and produce $'-Xarg\r' rather than -Xarg due to word + # splitting rules. + tr -s '\r\n' ' ' < "$1" + fi +} + +log() { + if [ "$MVNW_VERBOSE" = true ]; then + printf '%s\n' "$1" + fi +} + +BASE_DIR=$(find_maven_basedir "$(dirname "$0")") +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR +log "$MAVEN_PROJECTBASEDIR" + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" +if [ -r "$wrapperJarPath" ]; then + log "Found $wrapperJarPath" +else + log "Couldn't find $wrapperJarPath, downloading it ..." + + if [ -n "$MVNW_REPOURL" ]; then + wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + else + wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" + fi + while IFS="=" read -r key value; do + # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) + safeValue=$(echo "$value" | tr -d '\r') + case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; + esac + done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" + log "Downloading from: $wrapperUrl" + + if $cygwin; then + wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") + fi + + if command -v wget > /dev/null; then + log "Found wget ... using wget" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + else + wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + log "Found curl ... using curl" + [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + else + curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" + fi + else + log "Falling back to using Java to download" + javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" + javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaSource=$(cygpath --path --windows "$javaSource") + javaClass=$(cygpath --path --windows "$javaClass") + fi + if [ -e "$javaSource" ]; then + if [ ! -e "$javaClass" ]; then + log " - Compiling MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/javac" "$javaSource") + fi + if [ -e "$javaClass" ]; then + log " - Running MavenWrapperDownloader.java ..." + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +# If specified, validate the SHA-256 sum of the Maven wrapper jar file +wrapperSha256Sum="" +while IFS="=" read -r key value; do + case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; + esac +done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" +if [ -n "$wrapperSha256Sum" ]; then + wrapperSha256Result=false + if command -v sha256sum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + elif command -v shasum > /dev/null; then + if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then + wrapperSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." + echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + exit 1 + fi + if [ $wrapperSha256Result = false ]; then + echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 + echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 + echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + exit 1 + fi +fi + +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$JAVA_HOME" ] && + JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") + [ -n "$CLASSPATH" ] && + CLASSPATH=$(cygpath --path --windows "$CLASSPATH") + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +# shellcheck disable=SC2086 # safe args +exec "$JAVACMD" \ + $MAVEN_OPTS \ + $MAVEN_DEBUG_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/samples/semantickernel-demos/sk-database-sample/pom.xml b/samples/semantickernel-demos/sk-database-sample/pom.xml new file mode 100644 index 000000000..00640d596 --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/pom.xml @@ -0,0 +1,81 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + sk-database-sample + 1.4.4-RC3-SNAPSHOT + Database Sample + + + + + com.microsoft.semantic-kernel + semantickernel-bom + 1.4.4-RC2 + pom + import + + + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-mysql + + + com.microsoft.semantic-kernel + semantickernel-aiservices-openai + + + + org.slf4j + slf4j-api + + + com.mysql + mysql-connector-j + 9.1.0 + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 17 + 17 + + + + maven-assembly-plugin + + + + com.microsoft.semantickernel.Main + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/samples/semantickernel-demos/sk-database-sample/scripts/run.sh b/samples/semantickernel-demos/sk-database-sample/scripts/run.sh new file mode 100644 index 000000000..950feff33 --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/scripts/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# It takes some time for database to start, wait for it +sleep 10 + +set -a +. /run/secrets/ai-config +set +a + +java -jar sk-database-sample.jar \ No newline at end of file diff --git a/samples/semantickernel-demos/sk-database-sample/sk-database-sample-dockerfile b/samples/semantickernel-demos/sk-database-sample/sk-database-sample-dockerfile new file mode 100644 index 000000000..55a1d40ab --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/sk-database-sample-dockerfile @@ -0,0 +1,15 @@ +###################################################### +## Build Deployment +FROM mcr.microsoft.com/openjdk/jdk:21-ubuntu AS sk-database-sample-dockerfile + +COPY --chown=app:app scripts/run.sh /home/app/ + +RUN chmod +x /home/app/run.sh + +WORKDIR /home/app +USER app + +COPY target/sk-database-sample-*-jar-with-dependencies.jar /home/app/sk-database-sample.jar + +CMD /home/app/run.sh + diff --git a/samples/semantickernel-demos/sk-database-sample/src/main/java/com/microsoft/semantickernel/GitHubFile.java b/samples/semantickernel-demos/sk-database-sample/src/main/java/com/microsoft/semantickernel/GitHubFile.java new file mode 100644 index 000000000..fe60bf53d --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/src/main/java/com/microsoft/semantickernel/GitHubFile.java @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel; + +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; +import java.util.List; + +public class GitHubFile { + + @VectorStoreRecordKey + private final String id; + @VectorStoreRecordData + private final String description; + @VectorStoreRecordData + private final String link; + @VectorStoreRecordVector(dimensions = 1536, distanceFunction = DistanceFunction.COSINE_DISTANCE) + private final List embedding; + + public GitHubFile() { + this(null, null, null, Collections.emptyList()); + } + + public GitHubFile( + String id, + String description, + String link, + List embedding) { + this.id = id; + this.description = description; + this.link = link; + this.embedding = embedding; + } + + public String getId() { + return id; + } + + public String getDescription() { + return description; + } + + public String getLink() { + return link; + } + + public List getEmbedding() { + return embedding; + } + + public static String encodeId(String realId) { + byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); + return new String(bytes, StandardCharsets.UTF_8); + } +} \ No newline at end of file diff --git a/samples/semantickernel-demos/sk-database-sample/src/main/java/com/microsoft/semantickernel/Main.java b/samples/semantickernel-demos/sk-database-sample/src/main/java/com/microsoft/semantickernel/Main.java new file mode 100644 index 000000000..4eb507fc5 --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/src/main/java/com/microsoft/semantickernel/Main.java @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; +import com.mysql.cj.jdbc.MysqlDataSource; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class Main { + + private static final String USE_AZURE_CLIENT = System.getenv("USE_AZURE_CLIENT"); + + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = System.getenv() + .getOrDefault("MODEL_ID", "gpt-4o"); + + private static final String EMBEDDING_MODEL_ID = System.getenv() + .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); + + public static final String JDBC_MYSQL_TESTDB = "jdbc:mysql://mysql:3306/testdb"; + public static final String USERNAME = "a-user"; + public static final String PASSWORD = "a-password"; + + public static void main(String[] args) throws InterruptedException { + + OpenAIAsyncClient client; + + if (Boolean.parseBoolean(USE_AZURE_CLIENT)) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + Kernel kernel = buildKernel(client); + + // Create an OpenAI text embedding generation service + var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() + .withOpenAIAsyncClient(client) + .withModelId(EMBEDDING_MODEL_ID) + .withDimensions(1536) + .build(); + + JDBCVectorStore dataSource = buildDataSource(); + + VectorStoreRecordCollection collection = createCollection( + embeddingGeneration, dataSource); + + searchFor("How to get started", embeddingGeneration, collection); + searchFor("How do I create a react webapp", embeddingGeneration, + collection); + } + + public static VectorStoreRecordCollection createCollection( + OpenAITextEmbeddingGenerationService embeddingGeneration, + JDBCVectorStore jdbcVectorStore) { + + // Set up the record collection to use + String collectionName = "skgithubfiles"; + + var collection = jdbcVectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); + + // Create collection if it does not exist and store data + collection + .createCollectionIfNotExistsAsync() + .then(storeData(collection, embeddingGeneration, sampleData())) + .block(); + + return collection; + } + + private static void searchFor(String request, + OpenAITextEmbeddingGenerationService embeddingGeneration, + VectorStoreRecordCollection collection) { + System.out.println("-----------------------------------------------"); + System.out.println("Searching for '" + request + "' in the collection."); + + // Search for results + + var results = search(request, collection, embeddingGeneration).block(); + + if (results == null || results.getTotalCount() == 0) { + System.out.println("No search results found."); + return; + } + var searchResult = results.getResults().get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().getLink(), + searchResult.getRecord().getDescription()); + } + + + private static Mono> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); + } + + private static Mono> storeData( + VectorStoreRecordCollection recordStore, + OpenAITextEmbeddingGenerationService embeddingGeneration, + Map data) { + + return Flux.fromIterable(data.entrySet()) + .flatMap(entry -> { + System.out.println("Save '" + entry.getKey() + "' to memory."); + + // Generate embeddings for the data and store it + return embeddingGeneration + .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) + .flatMap(embeddings -> { + GitHubFile gitHubFile = new GitHubFile( + GitHubFile.encodeId(entry.getKey()), + entry.getValue(), + entry.getKey(), + embeddings.get(0).getVector()); + return recordStore.upsertAsync(gitHubFile, null); + }); + }) + .collectList(); + } + + + private static JDBCVectorStore buildDataSource() { + MysqlDataSource mysqlDataSource = new MysqlDataSource(); + mysqlDataSource.setUrl(JDBC_MYSQL_TESTDB); + mysqlDataSource.setUser(USERNAME); + mysqlDataSource.setPassword(PASSWORD); + MySQLVectorStoreQueryProvider queryProvider = MySQLVectorStoreQueryProvider.builder() + .withDataSource(mysqlDataSource) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(mysqlDataSource) + .withOptions( + JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build() + ) + .build(); + + vectorStore.prepareAsync().block(); + return vectorStore; + } + + private static Kernel buildKernel(OpenAIAsyncClient client) { + + ChatCompletionService chat = OpenAIChatCompletion.builder() + .withModelId(MODEL_ID) + .withOpenAIAsyncClient(client) + .build(); + + return Kernel + .builder() + .withAIService(ChatCompletionService.class, chat) + .build(); + + } + + private static Map sampleData() { + return Arrays.stream(new String[][]{ + {"https://github.com/microsoft/semantic-kernel/blob/main/README.md", + "README: Installation, getting started with Semantic Kernel, and how to contribute"}, + {"https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb", + "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function"}, + {"https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT", + "Sample demonstrating how to create a chat skill interfacing with ChatGPT"}, + {"https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs", + "C# class that defines a volatile embedding store"}, + {"https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md", + "README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4"}, + {"https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md", + "README: README associated with a sample chat summary react-based webapp"}, + }).collect(Collectors.toMap(element -> element[0], element -> element[1])); + } + +} diff --git a/samples/semantickernel-demos/sk-database-sample/src/main/resources/log4j2.xml b/samples/semantickernel-demos/sk-database-sample/src/main/resources/log4j2.xml new file mode 100644 index 000000000..50a638f0d --- /dev/null +++ b/samples/semantickernel-demos/sk-database-sample/src/main/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/semantickernel-demos/sk-presidio-sample/sk-presidio-sample-dockerfile b/samples/semantickernel-demos/sk-presidio-sample/sk-presidio-sample-dockerfile index 7a5102a23..cff0ef988 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/sk-presidio-sample-dockerfile +++ b/samples/semantickernel-demos/sk-presidio-sample/sk-presidio-sample-dockerfile @@ -1,6 +1,6 @@ ###################################################### ## Build Deployment -FROM mcr.microsoft.com/openjdk/jdk:21-ubuntu as presidio-sk-sample-app +FROM mcr.microsoft.com/openjdk/jdk:21-ubuntu AS presidio-sk-sample-app COPY --chown=app:app scripts/run.sh /home/app/ diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index 13bd19931..766829da2 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -44,10 +44,6 @@ jackson-core compile - - io.projectreactor - reactor-core -