Skip to content

Module: base_inference_workflow

Module implementing a base inference workflow.

This class is not meant to be subclassed directly; instead, subclass one of [TGIClientInferenceWorkflow, CSSInferenceWorkflow, BaseClassicInferenceWorkflow]

BaseInferenceWorkflow

Base class for an inference workflow

Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
class BaseInferenceWorkflow(metaclass=abc.ABCMeta):
    """
    Base class for an inference workflow
    """

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        """
        Constructor. keeps track of arguments passed in.
        """
        super().__init__()
        self.args: list[Any] = list(args)
        self.kwargs: dict[Any, Any] = kwargs

        self.is_setup = False
        self.__inference_count: int = 0
        self.__proof_count: int = 0

    def setup(self) -> Any:
        """
        calls setup and keeps track of whether or not setup was called
        """
        self.is_setup = True
        return self.do_setup()

    @abc.abstractmethod
    def do_setup(self) -> Any:
        """set up your workflow here.
        For LLMs, this may be parameters like top_k, temperature
        for classical LLM, this may be model hyperparams.

        Returns: Any
        """

    def stream(self, input_data: Any) -> Iterator[Any]:
        """
        Stream data for inference. Subclasses should implement do_stream.
        """
        logging.info("preprocessing input_data %s", input_data)
        preprocessed_data = self.do_preprocessing(input_data)
        yield from self.do_stream(preprocessed_data)

    def inference(self, input_data: Any) -> Any:
        """performs inference. Checks that model is set up before
        performing inference.
        Subclasses should implement do_inference.

        Args:
            input_data (typing.Any): input from user

        Raises:
            ValueError: if setup not called beforehand

        Returns:
            Any: result of inference
        """
        if not self.is_setup:
            raise ValueError("setup not called before inference")

        logging.info("preprocessing input_data %s", input_data)
        preprocessed_data = self.do_preprocessing(input_data)

        logging.info("querying model with %s", preprocessed_data)
        model_output = self.do_run_model(preprocessed_data)

        logging.info("postprocessing model_output %s", model_output)
        self.__inference_count += 1
        return self.do_postprocessing(input_data, model_output)

    @abc.abstractmethod
    def do_run_model(self, preprocessed_data: Any) -> Any:
        """run model here. preprocessed_data type is
            left generic since depending on model type
        Args:
            preprocessed_data (typing.Any): preprocessed input into model

        Returns:
            typing.Any: result of running model
        """
        pass

    def do_preprocessing(self, input_data: Any) -> Any:
        """
        Implement any preprocessing of the raw user input. For example, you may need to
        apply feature engineering on the input before it is suitable for model inference.
        By default, this method returns the input data as is.

        Args:
            input_data (Any): raw input from user

        Returns:
            Any: preprocessed input
        """
        return input_data

    @abc.abstractmethod
    def do_stream(self, preprocessed_input: Any) -> Iterator[Any]:
        """
        Implement any streaming logic here. For example, you may
        want to stream data from OpenAI or any LLM. This method
        should return the data to be streamed.

        Returns:
            typing.Any: data to be streamed
        """

    def do_postprocessing(self, input_data: Any, output_data: Any) -> Any:
        """
        Implement any postprocessing of the model output. By default, this method
        returns the output data as is.

        Args:
            input_data (Any): raw input from user
            output_data (Any): model output

        Returns:
            Any: postprocessed output
        """
        return output_data

    @final
    def generate_proof(self) -> None:
        """
        Generates proof. checks that setup performed before hand.
        """
        if self.__inference_count <= self.__proof_count:
            logging.warning(
                "generated %s inferences only but "
                + "already generated %s. Possibly duplicate proof.",
                self.__inference_count,
                self.__proof_count,
            )

        self.do_generate_proof()
        self.__proof_count += 1

    def do_generate_proof(self) -> Any:
        """
        Generates proof, which may vary based on proving system. We currently
        only support EZKL based proving, which does not require proof
        generation to be defined as part of the inference workflow if
        the inference from the circuit directly used for proving. Indeed,
        that is the case for the classic infernet_ml proof service. However, this
        may change with the usage of optimistic and other eager or lazy
        proof systems, which we intend to support in the future. By default,
        will raise NotImplementedError. Override in subclass as needed.
        """
        raise NotImplementedError

__init__(*args, **kwargs)

Constructor. keeps track of arguments passed in.

Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
21
22
23
24
25
26
27
28
29
30
31
def __init__(self, *args: Any, **kwargs: Any) -> None:
    """
    Constructor. keeps track of arguments passed in.
    """
    super().__init__()
    self.args: list[Any] = list(args)
    self.kwargs: dict[Any, Any] = kwargs

    self.is_setup = False
    self.__inference_count: int = 0
    self.__proof_count: int = 0

do_generate_proof()

Generates proof, which may vary based on proving system. We currently only support EZKL based proving, which does not require proof generation to be defined as part of the inference workflow if the inference from the circuit directly used for proving. Indeed, that is the case for the classic infernet_ml proof service. However, this may change with the usage of optimistic and other eager or lazy proof systems, which we intend to support in the future. By default, will raise NotImplementedError. Override in subclass as needed.

Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
151
152
153
154
155
156
157
158
159
160
161
162
def do_generate_proof(self) -> Any:
    """
    Generates proof, which may vary based on proving system. We currently
    only support EZKL based proving, which does not require proof
    generation to be defined as part of the inference workflow if
    the inference from the circuit directly used for proving. Indeed,
    that is the case for the classic infernet_ml proof service. However, this
    may change with the usage of optimistic and other eager or lazy
    proof systems, which we intend to support in the future. By default,
    will raise NotImplementedError. Override in subclass as needed.
    """
    raise NotImplementedError

do_postprocessing(input_data, output_data)

Implement any postprocessing of the model output. By default, this method returns the output data as is.

Parameters:

Name Type Description Default
input_data Any

raw input from user

 required
output_data Any

model output

 required

Returns:

Name Type Description
Any Any

postprocessed output
Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
def do_postprocessing(self, input_data: Any, output_data: Any) -> Any:
    """
    Implement any postprocessing of the model output. By default, this method
    returns the output data as is.

    Args:
        input_data (Any): raw input from user
        output_data (Any): model output

    Returns:
        Any: postprocessed output
    """
    return output_data

do_preprocessing(input_data)

Implement any preprocessing of the raw user input. For example, you may need to apply feature engineering on the input before it is suitable for model inference. By default, this method returns the input data as is.

Parameters:

Name Type Description Default
input_data Any

raw input from user

 required

Returns:

Name Type Description
Any Any

preprocessed input
Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
def do_preprocessing(self, input_data: Any) -> Any:
    """
    Implement any preprocessing of the raw user input. For example, you may need to
    apply feature engineering on the input before it is suitable for model inference.
    By default, this method returns the input data as is.

    Args:
        input_data (Any): raw input from user

    Returns:
        Any: preprocessed input
    """
    return input_data

do_run_model(preprocessed_data) abstractmethod

run model here. preprocessed_data type is left generic since depending on model type Args: preprocessed_data (typing.Any): preprocessed input into model

Returns:

Type Description
Any

typing.Any: result of running model
Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
84
85
86
87
88
89
90
91
92
93
94
@abc.abstractmethod
def do_run_model(self, preprocessed_data: Any) -> Any:
    """run model here. preprocessed_data type is
        left generic since depending on model type
    Args:
        preprocessed_data (typing.Any): preprocessed input into model

    Returns:
        typing.Any: result of running model
    """
    pass

do_setup() abstractmethod

set up your workflow here. For LLMs, this may be parameters like top_k, temperature for classical LLM, this may be model hyperparams.

Returns: Any

Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
40
41
42
43
44
45
46
47
@abc.abstractmethod
def do_setup(self) -> Any:
    """set up your workflow here.
    For LLMs, this may be parameters like top_k, temperature
    for classical LLM, this may be model hyperparams.

    Returns: Any
    """

do_stream(preprocessed_input) abstractmethod

Implement any streaming logic here. For example, you may want to stream data from OpenAI or any LLM. This method should return the data to be streamed.

Returns:

Type Description
Iterator[Any]

typing.Any: data to be streamed
Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
110
111
112
113
114
115
116
117
118
119
@abc.abstractmethod
def do_stream(self, preprocessed_input: Any) -> Iterator[Any]:
    """
    Implement any streaming logic here. For example, you may
    want to stream data from OpenAI or any LLM. This method
    should return the data to be streamed.

    Returns:
        typing.Any: data to be streamed
    """

generate_proof()

Generates proof. checks that setup performed before hand.

Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
@final
def generate_proof(self) -> None:
    """
    Generates proof. checks that setup performed before hand.
    """
    if self.__inference_count <= self.__proof_count:
        logging.warning(
            "generated %s inferences only but "
            + "already generated %s. Possibly duplicate proof.",
            self.__inference_count,
            self.__proof_count,
        )

    self.do_generate_proof()
    self.__proof_count += 1

inference(input_data)

performs inference. Checks that model is set up before performing inference. Subclasses should implement do_inference.

Parameters:

Name Type Description Default
input_data Any

input from user

 required

Raises:

Type Description
ValueError

if setup not called beforehand

Returns:

Name Type Description
Any Any

result of inference
Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def inference(self, input_data: Any) -> Any:
    """performs inference. Checks that model is set up before
    performing inference.
    Subclasses should implement do_inference.

    Args:
        input_data (typing.Any): input from user

    Raises:
        ValueError: if setup not called beforehand

    Returns:
        Any: result of inference
    """
    if not self.is_setup:
        raise ValueError("setup not called before inference")

    logging.info("preprocessing input_data %s", input_data)
    preprocessed_data = self.do_preprocessing(input_data)

    logging.info("querying model with %s", preprocessed_data)
    model_output = self.do_run_model(preprocessed_data)

    logging.info("postprocessing model_output %s", model_output)
    self.__inference_count += 1
    return self.do_postprocessing(input_data, model_output)

setup()

calls setup and keeps track of whether or not setup was called

Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
33
34
35
36
37
38
def setup(self) -> Any:
    """
    calls setup and keeps track of whether or not setup was called
    """
    self.is_setup = True
    return self.do_setup()

stream(input_data)

Stream data for inference. Subclasses should implement do_stream.

Source code in src/infernet_ml/workflows/inference/base_inference_workflow.py 
49
50
51
52
53
54
55
def stream(self, input_data: Any) -> Iterator[Any]:
    """
    Stream data for inference. Subclasses should implement do_stream.
    """
    logging.info("preprocessing input_data %s", input_data)
    preprocessed_data = self.do_preprocessing(input_data)
    yield from self.do_stream(preprocessed_data)