Skip to content

Commit b9d7def

Browse files
johnedquinnjohnedquinn
committed
Initializes performance benchmark suite
1 parent b694c6a commit b9d7def

File tree

14 files changed

+633
-0
lines changed

14 files changed

+633
-0
lines changed

‎settings.gradle.kts

+4
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,7 @@ include(
3636
"test:sprout-tests",
3737
"examples",
3838
)
39+
include("test:benchmarks")
40+
findProject(":test:benchmarks")?.name = "benchmarks"
41+
include("test:v1.0.0-perf.1")
42+
findProject(":test:v1.0.0-perf.1")?.name = "v1.0.0-perf.1"

‎test/benchmarks/README.adoc

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
= PartiQL Benchmarks
2+
3+
These benchmarks are meant for cross-version benchmarking.
4+
5+
== Prerequisites
6+
7+
[source, shell]
8+
.Terminal
9+
----
10+
git checkout v1.0.0-perf.1
11+
12+
# MANUALLY ADD
13+
# Add publishing plugin to the memory plugin
14+
# MANUALLY ADD
15+
16+
./gradlew publishToMavenLocal
17+
----
18+
19+
== Assemble & Run
20+
21+
To assemble:
22+
[source, shell]
23+
.Terminal
24+
----
25+
./gradlew :test:benchmarks:assemble
26+
----
27+
28+
To run:
29+
[source, shell]
30+
.Terminal
31+
----
32+
./gradlew :test:benchmarks:jmh
33+
----

‎test/benchmarks/build.gradle.kts

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
plugins {
17+
id(Plugins.conventions)
18+
id(Plugins.jmh) version Versions.jmhGradlePlugin
19+
id(Plugins.library)
20+
}
21+
22+
dependencies {
23+
implementation(project(":partiql-eval"))
24+
implementation(project(":partiql-planner"))
25+
implementation(project(":partiql-parser"))
26+
implementation(project(":plugins:partiql-memory"))
27+
implementation(project(path = ":test:v1.0.0-perf.1", configuration = "shadow"))
28+
}

‎test/benchmarks/src/jmh/kotlin/org/partiql/benchmarks/compiler/Compiler.kt

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package org.partiql.benchmarks.compiler
2+
3+
interface Compiler {
4+
fun compile(query: String): Iterable<Any>
5+
}

‎test/benchmarks/src/jmh/kotlin/org/partiql/benchmarks/compiler/CompilerBuilder.kt

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package org.partiql.benchmarks.compiler
2+
3+
class CompilerBuilder {
4+
private var version: Version = Version.EVAL_HEAD
5+
6+
fun current(): CompilerBuilder = this.apply {
7+
this.version = Version.EVAL_HEAD
8+
}
9+
10+
fun version(version: Version): CompilerBuilder {
11+
return this.apply {
12+
this.version = version
13+
}
14+
}
15+
16+
fun build(): Compiler {
17+
return when (this.version) {
18+
Version.EVAL_V1_0_0_PERF_1 -> CompilerV1_0_0_Perf_1()
19+
Version.EVAL_HEAD -> CompilerCurrent()
20+
Version.LEGACY_V1_0_0_PERF_1 -> CompilerLegacy()
21+
}
22+
}
23+
24+
enum class Version {
25+
EVAL_V1_0_0_PERF_1,
26+
EVAL_HEAD,
27+
LEGACY_V1_0_0_PERF_1,
28+
}
29+
}

‎test/benchmarks/src/jmh/kotlin/org/partiql/benchmarks/compiler/CompilerCurrent.kt

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package org.partiql.benchmarks.compiler
2+
3+
import org.partiql.eval.PartiQLEngine
4+
import org.partiql.eval.PartiQLResult
5+
import org.partiql.planner.catalog.Session
6+
import org.partiql.plugins.memory.MemoryCatalog
7+
import org.partiql.plugins.memory.MemoryConnector
8+
import org.partiql.spi.connector.ConnectorSession
9+
import org.partiql.value.CollectionValue
10+
import org.partiql.value.PartiQLValueExperimental
11+
12+
class CompilerCurrent : Compiler {
13+
14+
private val parser = org.partiql.parser.PartiQLParser.builder().build()
15+
16+
private val planner = org.partiql.planner.PartiQLPlanner.builder().build()
17+
18+
private val evaluator = PartiQLEngine.builder().build()
19+
20+
@OptIn(PartiQLValueExperimental::class)
21+
override fun compile(query: String): Iterable<Any> {
22+
val parseResult = parser.parse(query)
23+
val catalogName = "default"
24+
val catalog = MemoryCatalog.builder().name(catalogName)
25+
val connector = MemoryConnector(catalog.build())
26+
val connectorSession = object : ConnectorSession {
27+
override fun getQueryId(): String = "q"
28+
override fun getUserId(): String = "u"
29+
}
30+
val engineSession = PartiQLEngine.Session(mapOf(catalogName to connector))
31+
val plannerSession = Session.builder().catalog(catalogName).catalogs(catalogName to connector.getMetadata(connectorSession)).build()
32+
val planResult = planner.plan(parseResult.root, plannerSession)
33+
val statement = evaluator.prepare(planResult.plan, engineSession)
34+
val value = evaluator.execute(statement) as PartiQLResult.Value
35+
return value.value as CollectionValue<*>
36+
}
37+
}

‎test/benchmarks/src/jmh/kotlin/org/partiql/benchmarks/compiler/CompilerLegacy.kt

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package org.partiql.benchmarks.compiler
2+
3+
import org.partiql_v1_0_0_perf_1.lang.CompilerPipeline
4+
import org.partiql_v1_0_0_perf_1.lang.eval.EvaluationSession
5+
import org.partiql_v1_0_0_perf_1.lang.eval.PartiQLResult
6+
7+
class CompilerLegacy : Compiler {
8+
9+
private val compiler = CompilerPipeline.builder().build()
10+
11+
override fun compile(query: String): Iterable<Any> {
12+
val result = compiler.compile(query)
13+
val session = EvaluationSession.standard()
14+
val evaluationResult = result.evaluate(session)
15+
val value = evaluationResult as PartiQLResult.Value
16+
return value.value
17+
}
18+
}

‎test/benchmarks/src/jmh/kotlin/org/partiql/benchmarks/compiler/CompilerV1_0_0_Perf_1.kt

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package org.partiql.benchmarks.compiler
2+
3+
import org.partiql_v1_0_0_perf_1.value.CollectionValue
4+
import org.partiql_v1_0_0_perf_1.eval.PartiQLEngine
5+
import org.partiql_v1_0_0_perf_1.eval.PartiQLResult
6+
import org.partiql_v1_0_0_perf_1.parser.PartiQLParser
7+
import org.partiql_v1_0_0_perf_1.planner.PartiQLPlanner
8+
import org.partiql_v1_0_0_perf_1.plugins.memory.MemoryCatalog
9+
import org.partiql_v1_0_0_perf_1.plugins.memory.MemoryConnector
10+
import org.partiql_v1_0_0_perf_1.spi.connector.ConnectorSession
11+
import org.partiql_v1_0_0_perf_1.value.PartiQLValueExperimental
12+
13+
class CompilerV1_0_0_Perf_1 : Compiler {
14+
15+
private val parser = PartiQLParser.builder().build()
16+
17+
private val planner = PartiQLPlanner.builder().build()
18+
19+
private val evaluator = PartiQLEngine.builder().build()
20+
21+
@OptIn(PartiQLValueExperimental::class, org.partiql.value.PartiQLValueExperimental::class)
22+
override fun compile(query: String): Iterable<Any> {
23+
val parseResult = parser.parse(query)
24+
val catalogName = "default"
25+
val catalog = MemoryCatalog.builder().name(catalogName)
26+
val connector = MemoryConnector(catalog.build())
27+
val queryId = "q"
28+
val userId = "u"
29+
val connectorSession = object : ConnectorSession {
30+
override fun getQueryId(): String = queryId
31+
override fun getUserId(): String = userId
32+
}
33+
val engineSession = PartiQLEngine.Session(mapOf(catalogName to connector))
34+
val plannerSession = PartiQLPlanner.Session(queryId, userId, catalogName, catalogs = mapOf(catalogName to connector.getMetadata(connectorSession)))
35+
val planResult = planner.plan(parseResult.root, plannerSession)
36+
val statement = evaluator.prepare(planResult.plan, engineSession)
37+
val value = evaluator.execute(statement) as PartiQLResult.Value
38+
return value.value as CollectionValue<*>
39+
}
40+
}

‎test/benchmarks/src/jmh/kotlin/org/partiql/benchmarks/tests/BenchmarkConstants.kt

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright 2022 Amazon.com, Inc. or its affiliates. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at:
7+
*
8+
* http://aws.amazon.com/apache2.0/
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
12+
* language governing permissions and limitations under the License.
13+
*/
14+
15+
package org.partiql.benchmarks.tests
16+
17+
internal const val FORK_VALUE_RECOMMENDED: Int = 2
18+
internal const val MEASUREMENT_ITERATION_VALUE_RECOMMENDED: Int = 10
19+
internal const val MEASUREMENT_TIME_VALUE_RECOMMENDED: Int = 1
20+
internal const val WARMUP_ITERATION_VALUE_RECOMMENDED: Int = 5
21+
internal const val WARMUP_TIME_VALUE_RECOMMENDED: Int = 1

0 commit comments

Comments
 (0)