Skip to content

Commit c46cc5f

Browse files
tadayositadayosi
committed
Update README and add more examples
1 parent b0f47c3 commit c46cc5f

6 files changed

+306
-0
lines changed

‎README.md

+189
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ TorchServe Client for Java (TSC4J) is a Java client library for [TorchServe](htt
99
- [Management API](https://pytorch.org/serve/management_api.html)
1010
- [Metrics API](https://pytorch.org/serve/metrics_api.html)
1111

12+
## Requirements
13+
14+
- Java 17+
15+
1216
## Install
1317

1418
1. Add the [JitPack](https://jitpack.io) repository to your `pom.xml`:
@@ -34,6 +38,191 @@ TorchServe Client for Java (TSC4J) is a Java client library for [TorchServe](htt
3438

3539
## Usage
3640

41+
### Inference
42+
43+
- Prediction:
44+
45+
```java
46+
TorchServeClient client = TorchServeClient.newInstance();
47+
48+
byte[] image = Files.readAllBytes(Path.of("0.png"));
49+
Object result = client.inference().predictions("mnist_v2", image);
50+
System.out.println(result);
51+
// => 0
52+
```
53+
54+
- With the inference API endpoint other than <http://localhost:8080>:
55+
56+
```java
57+
TorchServeClient client = TorchServeClient.builder()
58+
.inferenceAddress("http://localhost:12345")
59+
.build();
60+
```
61+
62+
- With token authorization:
63+
64+
```java
65+
TorchServeClient client = TorchServeClient.builder()
66+
.inferenceKey("<inference-key>")
67+
.build();
68+
```
69+
70+
### Management
71+
72+
- Register a model:
73+
74+
```java
75+
TorchServeClient client = TorchServeClient.newInstance();
76+
77+
Response response = client.management().registerModel(
78+
"https://torchserve.pytorch.org/mar_files/mnist_v2.mar",
79+
RegisterModelOptions.empty());
80+
System.out.println(response.getStatus());
81+
// => "Model "mnist_v2" Version: 2.0 registered with 0 initial workers. Use scale workers API to add workers for the model."
82+
```
83+
84+
- Scale workers for a model:
85+
86+
```java
87+
TorchServeClient client = TorchServeClient.newInstance();
88+
89+
Response response = client.management().setAutoScale(
90+
"mnist_v2",
91+
SetAutoScaleOptions.builder()
92+
.minWorker(1)
93+
.maxWorker(2)
94+
.build());
95+
System.out.println(response.getStatus());
96+
// => "Processing worker updates..."
97+
```
98+
99+
- Describe a model:
100+
101+
```java
102+
TorchServeClient client = TorchServeClient.newInstance();
103+
104+
List<ModelDetail> model = client.management().describeModel("mnist_v2");
105+
System.out.println(model.get(0));
106+
// =>
107+
// ModelDetail {
108+
// modelName: mnist_v2
109+
// modelVersion: 2.0
110+
// ...
111+
```
112+
113+
- Unregister a model:
114+
115+
```java
116+
TorchServeClient client = TorchServeClient.newInstance();
117+
118+
Response response = client.management().unregisterModel(
119+
"mnist_v2",
120+
UnregisterModelOptions.empty());
121+
System.out.println(response.getStatus());
122+
// => "Model "mnist_v2" unregistered"
123+
```
124+
125+
- List models:
126+
127+
```java
128+
TorchServeClient client = TorchServeClient.newInstance();
129+
130+
ModelList models = client.management().listModels(10, null);
131+
System.out.println(models);
132+
// =>
133+
// ModelList {
134+
// nextPageToken: null
135+
// models: [Model {
136+
// modelName: mnist_v2
137+
// modelUrl: https://torchserve.pytorch.org/mar_files/mnist_v2.mar
138+
// },
139+
// ...
140+
```
141+
142+
- Set default version for a model:
143+
144+
```java
145+
TorchServeClient client = TorchServeClient.newInstance();
146+
147+
Response response = client.management().setDefault("mnist_v2", "2.0");
148+
System.out.println(response.getStatus());
149+
// => "Default version successfully updated for model "mnist_v2" to "2.0""
150+
```
151+
152+
- With the management API endpoint other than <http://localhost:8081>:
153+
154+
```java
155+
TorchServeClient client = TorchServeClient.builder()
156+
.managementAddress("http://localhost:12345")
157+
.build();
158+
```
159+
160+
- With token authorization:
161+
162+
```java
163+
TorchServeClient client = TorchServeClient.builder()
164+
.managementKey("<management-key>")
165+
.build();
166+
```
167+
168+
### Metrics
169+
170+
- Get metrics in Prometheus format:
171+
172+
```java
173+
TorchServeClient client = TorchServeClient.newInstance();
174+
175+
String metrics = client.metrics().metrics();
176+
System.out.println(metrics);
177+
// =>
178+
// # HELP MemoryUsed Torchserve prometheus gauge metric with unit: Megabytes
179+
// # TYPE MemoryUsed gauge
180+
// MemoryUsed{Level="Host",Hostname="3a9b51d41fbf",} 2075.09765625
181+
// ...
182+
```
183+
184+
- With the metrics API endpoint other than <http://localhost:8082>:
185+
186+
```java
187+
TorchServeClient client = TorchServeClient.builder()
188+
.metricsAddress("http://localhost:12345")
189+
.build();
190+
```
191+
192+
### Configuration
193+
194+
#### tsc4j.properties
195+
196+
```properties
197+
inference.key = <inference-key>
198+
inference.address = http://localhost:8080
199+
# inference.address takes precedence over inference.port if it's defined
200+
inference.port = 8080
201+
202+
management.key = <management-key>
203+
management.address = http://localhost:8081
204+
# management.address takes precedence over management.port if it's defined
205+
management.port = 8081
206+
207+
metrics.address = http://localhost:8082
208+
# metrics.address takes precedence over metrics.port if it's defined
209+
metrics.port = 8082
210+
```
211+
212+
#### System properties
213+
214+
You can configure the TSC4J properties via system properties with prefix `tsc4j.`.
215+
216+
For instance, you can configure `inference.address` with the `tsc4j.inference.address` system property.
217+
218+
#### Environment variables
219+
220+
You can also configure the TSC4J properties via environment variables with prefix `TSC4J_`.
221+
222+
For instance, you can configure `inference.address` with the `TSC4J_INFERENCE_ADDRESS` environment variable.
223+
224+
### Examples
225+
37226
See [examples](./examples/).
38227

39228
## Build

‎examples/describe_model.java

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
///usr/bin/env jbang "$0" "$@" ; exit $?
2+
3+
//JAVA 17+
4+
//REPOS mavencentral,jitpack=https://jitpack.io
5+
//DEPS com.github.tadayosi:torchserve-client-java:main-SNAPSHOT
6+
//DEPS org.slf4j:slf4j-simple:1.7.36
7+
8+
import com.github.tadayosi.torchserve.client.TorchServeClient;
9+
import com.github.tadayosi.torchserve.client.model.ApiException;
10+
11+
public class describe_model {
12+
13+
private static String MNIST_MODEL = "mnist_v2";
14+
15+
public static void main(String... args) throws Exception {
16+
try {
17+
var client = TorchServeClient.newInstance();
18+
var models = client.management().describeModel(MNIST_MODEL);
19+
System.out.println("Model> " + models.get(0));
20+
} catch (ApiException e) {
21+
e.printStackTrace();
22+
}
23+
}
24+
}

‎examples/list_models.java

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
///usr/bin/env jbang "$0" "$@" ; exit $?
2+
3+
//JAVA 17+
4+
//REPOS mavencentral,jitpack=https://jitpack.io
5+
//DEPS com.github.tadayosi:torchserve-client-java:main-SNAPSHOT
6+
//DEPS org.slf4j:slf4j-simple:1.7.36
7+
8+
import com.github.tadayosi.torchserve.client.TorchServeClient;
9+
import com.github.tadayosi.torchserve.client.model.ApiException;
10+
11+
public class list_models {
12+
13+
public static void main(String... args) throws Exception {
14+
try {
15+
var client = TorchServeClient.newInstance();
16+
var models = client.management().listModels(10, null);
17+
System.out.println("Models> " + models);
18+
} catch (ApiException e) {
19+
e.printStackTrace();
20+
}
21+
}
22+
}

‎examples/metrics.java

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
///usr/bin/env jbang "$0" "$@" ; exit $?
2+
3+
//JAVA 17+
4+
//REPOS mavencentral,jitpack=https://jitpack.io
5+
//DEPS com.github.tadayosi:torchserve-client-java:main-SNAPSHOT
6+
//DEPS org.slf4j:slf4j-simple:1.7.36
7+
8+
import com.github.tadayosi.torchserve.client.TorchServeClient;
9+
import com.github.tadayosi.torchserve.client.model.ApiException;
10+
11+
public class metrics {
12+
13+
public static void main(String... args) throws Exception {
14+
try {
15+
var client = TorchServeClient.newInstance();
16+
var metrics = client.metrics().metrics();
17+
System.out.println(metrics);
18+
} catch (ApiException e) {
19+
e.printStackTrace();
20+
}
21+
}
22+
}

‎examples/set_default.java

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
///usr/bin/env jbang "$0" "$@" ; exit $?
2+
3+
//JAVA 17+
4+
//REPOS mavencentral,jitpack=https://jitpack.io
5+
//DEPS com.github.tadayosi:torchserve-client-java:main-SNAPSHOT
6+
//DEPS org.slf4j:slf4j-simple:1.7.36
7+
8+
import com.github.tadayosi.torchserve.client.TorchServeClient;
9+
import com.github.tadayosi.torchserve.client.model.ApiException;
10+
11+
public class set_default {
12+
13+
private static String MNIST_MODEL = "mnist_v2";
14+
15+
public static void main(String... args) throws Exception {
16+
try {
17+
var client = TorchServeClient.newInstance();
18+
var response = client.management().setDefault(MNIST_MODEL, "2.0");
19+
System.out.println("Status> " + response.getStatus());
20+
} catch (ApiException e) {
21+
e.printStackTrace();
22+
}
23+
}
24+
}

‎examples/unregister_mnist.java

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
///usr/bin/env jbang "$0" "$@" ; exit $?
2+
3+
//JAVA 17+
4+
//REPOS mavencentral,jitpack=https://jitpack.io
5+
//DEPS com.github.tadayosi:torchserve-client-java:main-SNAPSHOT
6+
//DEPS org.slf4j:slf4j-simple:1.7.36
7+
8+
import com.github.tadayosi.torchserve.client.TorchServeClient;
9+
import com.github.tadayosi.torchserve.client.model.ApiException;
10+
import com.github.tadayosi.torchserve.client.model.UnregisterModelOptions;
11+
12+
public class unregister_mnist {
13+
14+
private static String MNIST_MODEL = "mnist_v2";
15+
16+
public static void main(String... args) throws Exception {
17+
try {
18+
var client = TorchServeClient.newInstance();
19+
var response = client.management().unregisterModel(MNIST_MODEL, UnregisterModelOptions.empty());
20+
System.out.println("Status> " + response.getStatus());
21+
} catch (ApiException e) {
22+
e.printStackTrace();
23+
}
24+
}
25+
}

0 commit comments

Comments
 (0)