Skip to content
This repository was archived by the owner on Feb 1, 2024. It is now read-only.

Commit ba912bc

Browse files
agoallikmaaagoallikmaa
authoredJun 20, 2022
Implement C3P0 connection pool metrics (open-telemetry#6174)
* C3P0 connection pool metrics * Use PooledDataSource instead of specific implementation * Add C3P0 readme * RuntimeException in case of underlying SQLException * Use ISE instead of RuntimeException
1 parent 11dd079 commit ba912bc

File tree

14 files changed

+516
-1
lines changed

14 files changed

+516
-1
lines changed
 

‎instrumentation/c3p0-0.9/javaagent/build.gradle.kts

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
plugins {
2+
id("otel.javaagent-instrumentation")
3+
}
4+
5+
muzzle {
6+
pass {
7+
group.set("com.mchange")
8+
module.set("c3p0")
9+
versions.set("[0.9.2,)")
10+
assertInverse.set(true)
11+
// these versions have missing dependencies in maven central
12+
skip("0.9.2-pre2-RELEASE", "0.9.2-pre3")
13+
}
14+
}
15+
16+
dependencies {
17+
library("com.mchange:c3p0:0.9.2")
18+
19+
implementation(project(":instrumentation:c3p0-0.9:library"))
20+
21+
testImplementation(project(":instrumentation:c3p0-0.9:testing"))
22+
}

‎instrumentation/c3p0-0.9/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/c3p0/AbstractPoolBackedDataSourceInstrumentation.java

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.javaagent.instrumentation.c3p0;
7+
8+
import static io.opentelemetry.javaagent.instrumentation.c3p0.C3p0Singletons.telemetry;
9+
import static net.bytebuddy.matcher.ElementMatchers.named;
10+
11+
import com.mchange.v2.c3p0.impl.AbstractPoolBackedDataSource;
12+
import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
13+
import io.opentelemetry.javaagent.extension.instrumentation.TypeTransformer;
14+
import net.bytebuddy.asm.Advice;
15+
import net.bytebuddy.description.type.TypeDescription;
16+
import net.bytebuddy.matcher.ElementMatcher;
17+
18+
final class AbstractPoolBackedDataSourceInstrumentation implements TypeInstrumentation {
19+
20+
@Override
21+
public ElementMatcher<TypeDescription> typeMatcher() {
22+
return named("com.mchange.v2.c3p0.impl.AbstractPoolBackedDataSource");
23+
}
24+
25+
@Override
26+
public void transform(TypeTransformer transformer) {
27+
transformer.applyAdviceToMethod(
28+
named("resetPoolManager"), this.getClass().getName() + "$ResetPoolManagerAdvice");
29+
transformer.applyAdviceToMethod(named("close"), this.getClass().getName() + "$CloseAdvice");
30+
}
31+
32+
@SuppressWarnings("unused")
33+
public static class ResetPoolManagerAdvice {
34+
35+
@Advice.OnMethodExit(suppress = Throwable.class)
36+
public static void onExit(@Advice.This AbstractPoolBackedDataSource dataSource) {
37+
telemetry().registerMetrics(dataSource);
38+
}
39+
}
40+
41+
@SuppressWarnings("unused")
42+
public static class CloseAdvice {
43+
44+
@Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class)
45+
public static void onExit(@Advice.This AbstractPoolBackedDataSource dataSource) {
46+
telemetry().unregisterMetrics(dataSource);
47+
}
48+
}
49+
}

‎instrumentation/c3p0-0.9/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/c3p0/C3p0InstrumentationModule.java

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.javaagent.instrumentation.c3p0;
7+
8+
import com.google.auto.service.AutoService;
9+
import io.opentelemetry.javaagent.extension.instrumentation.InstrumentationModule;
10+
import io.opentelemetry.javaagent.extension.instrumentation.TypeInstrumentation;
11+
import java.util.Collections;
12+
import java.util.List;
13+
14+
@AutoService(InstrumentationModule.class)
15+
public class C3p0InstrumentationModule extends InstrumentationModule {
16+
17+
public C3p0InstrumentationModule() {
18+
super("c3p0", "c3p0-0.9");
19+
}
20+
21+
@Override
22+
public List<TypeInstrumentation> typeInstrumentations() {
23+
return Collections.singletonList(new AbstractPoolBackedDataSourceInstrumentation());
24+
}
25+
}

‎instrumentation/c3p0-0.9/javaagent/src/main/java/io/opentelemetry/javaagent/instrumentation/c3p0/C3p0Singletons.java

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.javaagent.instrumentation.c3p0;
7+
8+
import io.opentelemetry.api.GlobalOpenTelemetry;
9+
import io.opentelemetry.instrumentation.c3p0.C3p0Telemetry;
10+
11+
public final class C3p0Singletons {
12+
13+
private static final C3p0Telemetry c3p0Telemetry =
14+
C3p0Telemetry.create(GlobalOpenTelemetry.get());
15+
16+
public static C3p0Telemetry telemetry() {
17+
return c3p0Telemetry;
18+
}
19+
20+
private C3p0Singletons() {}
21+
}

‎instrumentation/c3p0-0.9/javaagent/src/test/java/io/opentelemetry/javaagent/instrumentation/c3p0/C3p0InstrumentationTest.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.javaagent.instrumentation.c3p0;
7+
8+
import com.mchange.v2.c3p0.PooledDataSource;
9+
import io.opentelemetry.instrumentation.c3p0.AbstractC3p0InstrumentationTest;
10+
import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
11+
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
12+
import org.junit.jupiter.api.extension.RegisterExtension;
13+
14+
public class C3p0InstrumentationTest extends AbstractC3p0InstrumentationTest {
15+
16+
@RegisterExtension
17+
static final InstrumentationExtension testing = AgentInstrumentationExtension.create();
18+
19+
@Override
20+
protected InstrumentationExtension testing() {
21+
return testing;
22+
}
23+
24+
@Override
25+
protected void configure(PooledDataSource dataSource) {}
26+
27+
@Override
28+
protected void shutdown(PooledDataSource dataSource) {}
29+
}

‎instrumentation/c3p0-0.9/library/README.md

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Manual Instrumentation for C3P0
2+
3+
Provides OpenTelemetry instrumentation for [C3P0](https://www.mchange.com/projects/c3p0/).
4+
5+
## Quickstart
6+
7+
### Add these dependencies to your project:
8+
9+
Replace `OPENTELEMETRY_VERSION` with the latest stable
10+
[release](https://mvnrepository.com/artifact/io.opentelemetry). `Minimum version: 1.15.0`
11+
12+
For Maven, add to your `pom.xml` dependencies:
13+
14+
```xml
15+
16+
<dependencies>
17+
<dependency>
18+
<groupId>io.opentelemetry.instrumentation</groupId>
19+
<artifactId>opentelemetry-c3p0-0.9</artifactId>
20+
<version>OPENTELEMETRY_VERSION</version>
21+
</dependency>
22+
</dependencies>
23+
```
24+
25+
For Gradle, add to your dependencies:
26+
27+
```groovy
28+
implementation("io.opentelemetry.instrumentation:opentelemetry-c3p0-0.9:OPENTELEMETRY_VERSION")
29+
```
30+
31+
### Usage
32+
33+
The instrumentation library allows registering `PooledDataSource` instances for
34+
collecting OpenTelemetry-based metrics.
35+
36+
```java
37+
C3p0Telemetry c3p0Telemetry;
38+
39+
void configure(OpenTelemetry openTelemetry, PooledDataSource dataSource) {
40+
c3p0Telemetry = C3p0Telemetry.create(openTelemetry);
41+
c3p0Telemetry.registerMetrics(dataSource);
42+
}
43+
44+
void destroy(PooledDataSource dataSource) {
45+
c3p0Telemetry.unregisterMetrics(dataSource);
46+
}
47+
```

‎instrumentation/c3p0-0.9/library/build.gradle.kts

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
plugins {
2+
id("otel.library-instrumentation")
3+
id("otel.nullaway-conventions")
4+
}
5+
6+
dependencies {
7+
library("com.mchange:c3p0:0.9.2")
8+
9+
testImplementation(project(":instrumentation:c3p0-0.9:testing"))
10+
}

‎instrumentation/c3p0-0.9/library/src/main/java/io/opentelemetry/instrumentation/c3p0/C3p0Telemetry.java

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.c3p0;
7+
8+
import com.mchange.v2.c3p0.PooledDataSource;
9+
import io.opentelemetry.api.OpenTelemetry;
10+
11+
public final class C3p0Telemetry {
12+
/** Returns a new {@link C3p0Telemetry} configured with the given {@link OpenTelemetry}. */
13+
public static C3p0Telemetry create(OpenTelemetry openTelemetry) {
14+
return new C3p0Telemetry(openTelemetry);
15+
}
16+
17+
private final OpenTelemetry openTelemetry;
18+
19+
private C3p0Telemetry(OpenTelemetry openTelemetry) {
20+
this.openTelemetry = openTelemetry;
21+
}
22+
23+
/** Start collecting metrics for given connection pool. */
24+
public void registerMetrics(PooledDataSource dataSource) {
25+
ConnectionPoolMetrics.registerMetrics(openTelemetry, dataSource);
26+
}
27+
28+
/** Stop collecting metrics for given connection pool. */
29+
public void unregisterMetrics(PooledDataSource dataSource) {
30+
ConnectionPoolMetrics.unregisterMetrics(dataSource);
31+
}
32+
}

‎instrumentation/c3p0-0.9/library/src/main/java/io/opentelemetry/instrumentation/c3p0/ConnectionPoolMetrics.java

+122
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.c3p0;
7+
8+
import com.mchange.v2.c3p0.PooledDataSource;
9+
import io.opentelemetry.api.OpenTelemetry;
10+
import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;
11+
import io.opentelemetry.instrumentation.api.metrics.db.DbConnectionPoolMetrics;
12+
import java.sql.SQLException;
13+
import java.util.Arrays;
14+
import java.util.List;
15+
import java.util.Map;
16+
import java.util.concurrent.ConcurrentHashMap;
17+
import java.util.function.LongSupplier;
18+
import javax.annotation.Nullable;
19+
20+
final class ConnectionPoolMetrics {
21+
private static final String INSTRUMENTATION_NAME = "io.opentelemetry.c3p0-0.9";
22+
23+
// a weak map does not make sense here because each Meter holds a reference to the dataSource
24+
// PooledDataSource implements equals() & hashCode() in IdentityTokenResolvable,
25+
// that's why we wrap it with IdentityDataSourceKey that uses identity comparison instead
26+
private static final Map<IdentityDataSourceKey, List<ObservableLongUpDownCounter>>
27+
dataSourceMetrics = new ConcurrentHashMap<>();
28+
29+
public static void registerMetrics(OpenTelemetry openTelemetry, PooledDataSource dataSource) {
30+
dataSourceMetrics.compute(
31+
new IdentityDataSourceKey(dataSource),
32+
(key, existingCounters) ->
33+
ConnectionPoolMetrics.createMeters(openTelemetry, key, existingCounters));
34+
}
35+
36+
private static List<ObservableLongUpDownCounter> createMeters(
37+
OpenTelemetry openTelemetry,
38+
IdentityDataSourceKey key,
39+
List<ObservableLongUpDownCounter> existingCounters) {
40+
// remove old counters from the registry in case they were already there
41+
removeMetersFromRegistry(existingCounters);
42+
43+
PooledDataSource dataSource = key.dataSource;
44+
45+
DbConnectionPoolMetrics metrics =
46+
DbConnectionPoolMetrics.create(
47+
openTelemetry, INSTRUMENTATION_NAME, dataSource.getDataSourceName());
48+
49+
return Arrays.asList(
50+
metrics.usedConnections(wrapThrowingSupplier(dataSource::getNumBusyConnectionsDefaultUser)),
51+
metrics.idleConnections(wrapThrowingSupplier(dataSource::getNumIdleConnectionsDefaultUser)),
52+
metrics.pendingRequestsForConnection(
53+
wrapThrowingSupplier(dataSource::getNumThreadsAwaitingCheckoutDefaultUser)));
54+
}
55+
56+
public static void unregisterMetrics(PooledDataSource dataSource) {
57+
List<ObservableLongUpDownCounter> meters =
58+
dataSourceMetrics.remove(new IdentityDataSourceKey(dataSource));
59+
removeMetersFromRegistry(meters);
60+
}
61+
62+
private static void removeMetersFromRegistry(
63+
@Nullable List<ObservableLongUpDownCounter> observableInstruments) {
64+
if (observableInstruments != null) {
65+
for (ObservableLongUpDownCounter observable : observableInstruments) {
66+
observable.close();
67+
}
68+
}
69+
}
70+
71+
/**
72+
* A wrapper over {@link PooledDataSource} that implements identity comparison in its {@link
73+
* #equals(Object)} and {@link #hashCode()} methods.
74+
*/
75+
static final class IdentityDataSourceKey {
76+
final PooledDataSource dataSource;
77+
78+
IdentityDataSourceKey(PooledDataSource dataSource) {
79+
this.dataSource = dataSource;
80+
}
81+
82+
@Override
83+
@SuppressWarnings("ReferenceEquality")
84+
public boolean equals(@Nullable Object o) {
85+
if (this == o) {
86+
return true;
87+
}
88+
if (o == null || getClass() != o.getClass()) {
89+
return false;
90+
}
91+
IdentityDataSourceKey that = (IdentityDataSourceKey) o;
92+
return dataSource == that.dataSource;
93+
}
94+
95+
@Override
96+
public int hashCode() {
97+
return System.identityHashCode(dataSource);
98+
}
99+
100+
@Override
101+
public String toString() {
102+
return dataSource.toString();
103+
}
104+
}
105+
106+
static LongSupplier wrapThrowingSupplier(DataSourceIntSupplier supplier) {
107+
return () -> {
108+
try {
109+
return supplier.getAsInt();
110+
} catch (SQLException e) {
111+
throw new IllegalStateException("Failed to get C3P0 datasource metric", e);
112+
}
113+
};
114+
}
115+
116+
@FunctionalInterface
117+
interface DataSourceIntSupplier {
118+
int getAsInt() throws SQLException;
119+
}
120+
121+
private ConnectionPoolMetrics() {}
122+
}

‎instrumentation/c3p0-0.9/library/src/test/java/io/opentelemetry/instrumentation/c3p0/C3p0InstrumentationTest.java

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.c3p0;
7+
8+
import com.mchange.v2.c3p0.PooledDataSource;
9+
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
10+
import io.opentelemetry.instrumentation.testing.junit.LibraryInstrumentationExtension;
11+
import org.junit.jupiter.api.BeforeAll;
12+
import org.junit.jupiter.api.extension.RegisterExtension;
13+
14+
public class C3p0InstrumentationTest extends AbstractC3p0InstrumentationTest {
15+
16+
@RegisterExtension
17+
static final InstrumentationExtension testing = LibraryInstrumentationExtension.create();
18+
19+
private static C3p0Telemetry telemetry;
20+
21+
@Override
22+
protected InstrumentationExtension testing() {
23+
return testing;
24+
}
25+
26+
@BeforeAll
27+
static void setup() {
28+
telemetry = C3p0Telemetry.create(testing.getOpenTelemetry());
29+
}
30+
31+
@Override
32+
protected void configure(PooledDataSource dataSource) {
33+
telemetry.registerMetrics(dataSource);
34+
}
35+
36+
@Override
37+
protected void shutdown(PooledDataSource dataSource) {
38+
telemetry.unregisterMetrics(dataSource);
39+
}
40+
}

‎instrumentation/c3p0-0.9/testing/build.gradle.kts

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
plugins {
2+
id("otel.java-conventions")
3+
}
4+
5+
dependencies {
6+
api(project(":testing-common"))
7+
api("org.mockito:mockito-core")
8+
api("org.mockito:mockito-junit-jupiter")
9+
10+
compileOnly("com.mchange:c3p0:0.9.2")
11+
}

‎instrumentation/c3p0-0.9/testing/src/main/java/io/opentelemetry/instrumentation/c3p0/AbstractC3p0InstrumentationTest.java

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright The OpenTelemetry Authors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package io.opentelemetry.instrumentation.c3p0;
7+
8+
import static org.assertj.core.api.Assertions.assertThat;
9+
10+
import com.mchange.v2.c3p0.ComboPooledDataSource;
11+
import com.mchange.v2.c3p0.PooledDataSource;
12+
import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
13+
import io.opentelemetry.instrumentation.testing.junit.db.DbConnectionPoolMetricsAssertions;
14+
import io.opentelemetry.instrumentation.testing.junit.db.MockDriver;
15+
import java.sql.Connection;
16+
import java.sql.SQLException;
17+
import java.util.Arrays;
18+
import java.util.HashSet;
19+
import java.util.Set;
20+
import java.util.concurrent.TimeUnit;
21+
import org.junit.jupiter.api.BeforeAll;
22+
import org.junit.jupiter.api.Test;
23+
import org.junit.jupiter.api.extension.ExtendWith;
24+
import org.mockito.junit.jupiter.MockitoExtension;
25+
26+
@ExtendWith(MockitoExtension.class)
27+
public abstract class AbstractC3p0InstrumentationTest {
28+
private static final String INSTRUMENTATION_NAME = "io.opentelemetry.c3p0-0.9";
29+
30+
protected abstract InstrumentationExtension testing();
31+
32+
protected abstract void configure(PooledDataSource dataSource) throws Exception;
33+
34+
protected abstract void shutdown(PooledDataSource dataSource) throws Exception;
35+
36+
@BeforeAll
37+
static void setUpMocks() throws SQLException {
38+
MockDriver.register();
39+
}
40+
41+
@Test
42+
void shouldReportMetrics() throws Exception {
43+
// given
44+
ComboPooledDataSource c3p0DataSource = new ComboPooledDataSource();
45+
c3p0DataSource.setDriverClass(MockDriver.class.getName());
46+
c3p0DataSource.setJdbcUrl("jdbc:mock:testDatabase");
47+
48+
// when
49+
Connection connection = c3p0DataSource.getConnection();
50+
configure(c3p0DataSource);
51+
TimeUnit.MILLISECONDS.sleep(100);
52+
connection.close();
53+
54+
// then
55+
assertDataSourceMetrics(c3p0DataSource);
56+
57+
// when
58+
shutdown(c3p0DataSource);
59+
c3p0DataSource.close();
60+
61+
// wait interval of the test metrics exporter
62+
Thread.sleep(100);
63+
testing().clearData();
64+
Thread.sleep(100);
65+
66+
// then
67+
Set<String> metricNames =
68+
new HashSet<>(
69+
Arrays.asList("db.client.connections.usage", "db.client.connections.pending_requests"));
70+
assertThat(testing().metrics())
71+
.filteredOn(
72+
metricData ->
73+
metricData.getInstrumentationScopeInfo().getName().equals(INSTRUMENTATION_NAME)
74+
&& metricNames.contains(metricData.getName()))
75+
.isEmpty();
76+
}
77+
78+
private void assertDataSourceMetrics(PooledDataSource dataSource) {
79+
String dataSourceName = dataSource.getDataSourceName();
80+
81+
assertThat(dataSourceName)
82+
.as("c3p0 generates a unique pool name if it's not explicitly provided")
83+
.isNotEmpty();
84+
85+
DbConnectionPoolMetricsAssertions.create(
86+
testing(), INSTRUMENTATION_NAME, dataSource.getDataSourceName())
87+
.disableMinIdleConnections()
88+
.disableMaxIdleConnections()
89+
.disableMaxConnections()
90+
.disableConnectionTimeouts()
91+
.disableCreateTime()
92+
.disableWaitTime()
93+
.disableUseTime()
94+
.assertConnectionPoolEmitsMetrics();
95+
}
96+
}

‎settings.gradle.kts

+3
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,9 @@ include(":instrumentation:internal:internal-reflection:javaagent")
187187
include(":instrumentation:internal:internal-reflection:javaagent-integration-tests")
188188
include(":instrumentation:internal:internal-url-class-loader:javaagent")
189189
include(":instrumentation:internal:internal-url-class-loader:javaagent-integration-tests")
190+
include(":instrumentation:c3p0-0.9:javaagent")
191+
include(":instrumentation:c3p0-0.9:library")
192+
include(":instrumentation:c3p0-0.9:testing")
190193
include(":instrumentation:couchbase:couchbase-2.0:javaagent")
191194
include(":instrumentation:couchbase:couchbase-2.6:javaagent")
192195
include(":instrumentation:couchbase:couchbase-2-common:javaagent")

‎testing-common/src/main/java/io/opentelemetry/instrumentation/testing/junit/db/DbConnectionPoolMetricsAssertions.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public static DbConnectionPoolMetricsAssertions create(
3232

3333
private boolean testMinIdleConnections = true;
3434
private boolean testMaxIdleConnections = true;
35+
private boolean testMaxConnections = true;
3536
private boolean testPendingRequests = true;
3637
private boolean testConnectionTimeouts = true;
3738
private boolean testCreateTime = true;
@@ -55,6 +56,11 @@ public DbConnectionPoolMetricsAssertions disableMaxIdleConnections() {
5556
return this;
5657
}
5758

59+
public DbConnectionPoolMetricsAssertions disableMaxConnections() {
60+
testMaxConnections = false;
61+
return this;
62+
}
63+
5864
public DbConnectionPoolMetricsAssertions disablePendingRequests() {
5965
testPendingRequests = false;
6066
return this;
@@ -88,7 +94,9 @@ public void assertConnectionPoolEmitsMetrics() {
8894
if (testMaxIdleConnections) {
8995
verifyMaxIdleConnections();
9096
}
91-
verifyMaxConnections();
97+
if (testMaxConnections) {
98+
verifyMaxConnections();
99+
}
92100
if (testPendingRequests) {
93101
verifyPendingRequests();
94102
}

0 commit comments

Comments
 (0)
This repository has been archived.