import json
from typing import Dict, Any
from mlserver import MLModel, types
from mlserver.codecs import StringCodec
class EchoModel(MLModel):
async def load(self) -> bool:
print("Echo Model Initialized")
return await super().load()
async def predict(self, payload: types.InferenceRequest) -> types.InferenceResponse:
return types.InferenceResponse(
id=payload.id,
model_name=self.name,
model_version=self.version,
outputs=[
types.ResponseOutput(
name=input.name,
shape=input.shape,
datatype=input.datatype,
data=input.data,
parameters=input.parameters,
)
for input in payload.inputs
],
){
"inputs": [
{
"name": "fp32-input",
"shape": [1, 1],
"datatype": "FP32",
"data": [ 1.23 ]
},
{
"name": "int32-input",
"shape": [1, 1],
"datatype": "INT32",
"data": [ 1.23 ]
},
{
"name": "bytes-input",
"shape": [1, 1],
"datatype": "BYTES",
"data": [ "some bytes" ]
}
],
"outputs": [
{
"name": "predict"
}
]
}{"inputs":[{"name":"fp32-input","shape":[1,1],"datatype":"FP32","data":[1.23]},{"name":"int32-input","shape":[1,1],"datatype":"INT32","data":[1.23]},{"name":"bytes-input","shape":[1,1],"datatype":"BYTES","data":["some bytes"]}],"outputs":[{"name":"predict"}]}
{"inputs":[{"name":"fp32-input","shape":[1,1],"datatype":"FP32","data":[1.23]},{"name":"int32-input","shape":[1,1],"datatype":"INT32","data":[1.23]},{"name":"bytes-input","shape":[1,1],"datatype":"BYTES","data":["some bytes"]}],"outputs":[{"name":"predict"}]}
{"inputs":[{"name":"fp32-input","shape":[1,1],"datatype":"FP32","data":[1.23]},{"name":"int32-input","shape":[1,1],"datatype":"INT32","data":[1.23]},{"name":"bytes-input","shape":[1,1],"datatype":"BYTES","data":["some bytes"]}],"outputs":[{"name":"predict"}]}
{"inputs":[{"name":"fp32-input","shape":[1,1],"datatype":"FP32","data":[1.23]},{"name":"int32-input","shape":[1,1],"datatype":"INT32","data":[1.23]},{"name":"bytes-input","shape":[1,1],"datatype":"BYTES","data":["some bytes"]}],"outputs":[{"name":"predict"}]}
{"inputs":[{"name":"fp32-input","shape":[1,1],"datatype":"FP32","data":[1.23]},{"name":"int32-input","shape":[1,1],"datatype":"INT32","data":[1.23]},{"name":"bytes-input","shape":[1,1],"datatype":"BYTES","data":["some bytes"]}],"outputs":[{"name":"predict"}]}
2023-06-12 20:52:04,787 [mlserver] ERROR - consumer 0: failed to preprocess items: got unexpected datatype None from numpy array, expected BYTES
mlserver infer ...subcommand does not support BYTES inputs - reproduced by mock v2 echo model here.I used a simple Echo Model for testing
with
settings.json{ "debug": "true" }and
model-settings.json{ "name": "echo-model", "implementation": "model.EchoModel" }the model served locally with
mlserver startdoes properly process request of form{ "inputs": [ { "name": "fp32-input", "shape": [1, 1], "datatype": "FP32", "data": [ 1.23 ] }, { "name": "int32-input", "shape": [1, 1], "datatype": "INT32", "data": [ 1.23 ] }, { "name": "bytes-input", "shape": [1, 1], "datatype": "BYTES", "data": [ "some bytes" ] } ], "outputs": [ { "name": "predict" } ] }however when trying to put it as input batch with
and run via
I am getting